Merge pull request #152 from mempool/mononaut/enable-liquid-asset-registry

enable liquid asset registry in start script

.dockerignore

@@ -0,0 +1 @@
+Dockerfile

.github/actions/ci-rust-setup/action.yml

@@ -0,0 +1,50 @@
+name: CI Rust Setup
+description: 'Sets up the environment for Rust jobs during CI workflow'
+
+inputs:
+  cache-name:
+    description: 'Name of cache artifacts (same name is same cache key) empty to disable cache'
+    required: false
+  targets:
+    description: 'A comma separated list of extra targets you want to install'
+    required: false
+  components:
+    description: 'A comma separated list of extra components you want to install'
+    required: false
+  toolchain:
+    description: 'The toolchain to use. If not specified, the rust-toolchain file will be used'
+    required: false
+
+runs:
+  using: composite
+  steps:
+    - name: Get toolchain from input OR rust-toolchain file
+      id: gettoolchain
+      shell: bash
+      run: |-
+        RUST_TOOLCHAIN="${{ inputs.toolchain }}"
+        if [ ! -f rust-toolchain ] && [ -z "${RUST_TOOLCHAIN}" ]; then
+          echo "***ERROR*** NEED toolchain INPUT OR rust-toolchain FILE IN ROOT OF REPOSITORY" >&2
+          exit 1
+        fi
+        if [ -z "${RUST_TOOLCHAIN}" ]; then
+          RUST_TOOLCHAIN="$(cat rust-toolchain)"
+        fi
+        echo "toolchain=\"${RUST_TOOLCHAIN}\"" >> $GITHUB_OUTPUT
+    - name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
+      id: toolchain
+      # Commit date is Nov 18, 2024
+      uses: dtolnay/rust-toolchain@315e265cd78dad1e1dcf3a5074f6d6c47029d5aa
+      with:
+        toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
+        targets: ${{ inputs.targets }}
+        components: ${{ inputs.components }}
+    - name: Cache dependencies
+      uses: actions/cache@v3
+      if: inputs.cache-name != ''
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target
+        key: ${{ runner.os }}-cargo-${{ inputs.cache-name }}-${{ steps.toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock') }}

.github/workflows/ci.yml

@@ -0,0 +1,107 @@
+on:
+  pull_request:
+  push:
+    branches:
+      - mempool
+
+
+name: Compile Check and Lint
+
+jobs:
+  check:
+    name: Compile Check
+    runs-on: mempool-ci
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust
+        uses: './.github/actions/ci-rust-setup'
+        with:
+          cache-name: dev
+      - run: cargo check --all-features
+
+  fmt:
+    name: Formatter
+    runs-on: mempool-ci
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust
+        uses: './.github/actions/ci-rust-setup'
+        with:
+          components: rustfmt
+      - run: cargo fmt --all -- --check
+
+  test:
+    name: Run Tests
+    runs-on: mempool-ci
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust
+        uses: './.github/actions/ci-rust-setup'
+        with:
+          cache-name: test
+      - run: cargo test --lib --all-features
+
+  compile-freebsd:
+    runs-on: mempool-ci
+    name: Run Compile Checks in FreeBSD
+    env:
+      FREEBSD_VER: "14.3"
+    steps:
+    - uses: actions/checkout@v4
+    - name: Cache dependencies for FreeBSD
+      uses: actions/cache@v3
+      with:
+        path: |
+          .cargohome/registry
+          .cargohome/git
+          target
+        key: freebsd-${{ env.FREEBSD_VER }}-cargo-checks-${{ hashFiles('**/Cargo.lock') }}
+    - name: Compile Checks in FreeBSD
+      uses: vmactions/freebsd-vm@v1
+      with:
+        usesh: true
+        release: "${{ env.FREEBSD_VER }}"
+        arch: amd64
+        prepare: |
+          mkdir -p ~/.cargo/
+          mkdir -p ./.cargohome/registry/
+          mkdir -p ./.cargohome/git/
+          mv ./.cargohome/registry ~/.cargo/
+          mv ./.cargohome/git ~/.cargo/
+          rm -rf ./.cargohome
+          pkg install -y git rsync gmake llvm rust rocksdb cmake
+
+        run: |
+          cargo check --no-default-features
+          cargo check -F liquid
+          cargo check -F electrum-discovery
+          cargo check -F electrum-discovery,liquid
+          cargo build --release --bin electrs
+          rm -rf ./.cargohome
+          mkdir -p ~/.cargo/registry/
+          mkdir -p ~/.cargo/git/
+          mkdir -p ./.cargohome/
+          mv ~/.cargo/registry ./.cargohome/
+          mv ~/.cargo/git ./.cargohome/
+
+  clippy:
+    name: Linter
+    runs-on: mempool-ci
+    needs: [check]
+    strategy:
+      matrix: # Try all combinations of features. Some times weird things appear.
+        features: [
+          '',
+          '-F electrum-discovery',
+          '-F liquid',
+          '-F electrum-discovery,liquid',
+        ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust
+        uses: './.github/actions/ci-rust-setup'
+        with:
+          cache-name: dev
+          components: clippy
+      - name: Clippy with Features = ${{ matrix.features }}
+        run: cargo clippy ${{ matrix.features }} -- -D warnings

.github/workflows/on-tag.yml

@@ -0,0 +1,77 @@
+name: Docker build on tag
+env:
+  DOCKER_CLI_EXPERIMENTAL: enabled
+  TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
+  DOCKER_BUILDKIT: 0
+  COMPOSE_DOCKER_CLI_BUILD: 0
+
+on:
+  push:
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+      - v[0-9]+.[0-9]+.[0-9]+-*
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: mempool-ci
+    timeout-minutes: 120
+    name: Build and push to DockerHub
+    strategy:
+      max-parallel: 1
+      matrix:
+        include:
+          - image: electrs
+            cargo_extra_args: ""
+          - image: electrs-liquid
+            cargo_extra_args: "--features liquid"
+    steps:
+      - name: Set env variables
+        run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
+
+      - name: Show set environment variables
+        run: |
+          printf "      TAG: %s\n"  "$TAG"
+
+      - name: Add SHORT_SHA env property with commit short sha
+        run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
+
+      - name: Login to Docker for building
+        run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
+
+      - name: Checkout project
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        id: qemu
+
+      - name: Setup Docker buildx action
+        uses: docker/setup-buildx-action@v3
+        id: buildx
+
+      - name: Available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        id: cache
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx
+          restore-keys: |
+            ${{ runner.os }}-buildx
+
+      - name: Run Docker buildx against tag
+        run: |
+          docker buildx build \
+          --cache-from "type=local,src=/tmp/.buildx-cache" \
+          --cache-to "type=local,dest=/tmp/.buildx-cache" \
+          --platform linux/amd64,linux/arm64 \
+          --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.image }}:$TAG \
+          --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.image }}:latest \
+          --output "type=registry" . \
+          --build-arg commitHash=$SHORT_SHA \
+          --build-arg CARGO_EXTRA_ARGS="${{ matrix.cargo_extra_args }}"

.github/workflows/project-review-status.yml

@@ -0,0 +1,18 @@
+name: Project Board Automation
+
+on:
+  pull_request:
+    types: [review_requested]
+  issues:
+    types: [opened]
+
+jobs:
+  project-automation:
+    uses: mempool/.github/.github/workflows/project-board-automation.yml@master
+    with:
+      project-number: 8
+    secrets:
+      PROJECT_TOKEN: ${{ secrets.PROJECT_TOKEN }}
+      PROJECT_ID: ${{ secrets.PROJECT_ID }}
+      STATUS_FIELD_ID: ${{ secrets.STATUS_FIELD_ID }}
+      REVIEW_NEEDED_OPTION_ID: ${{ secrets.REVIEW_NEEDED_OPTION_ID }}

.gitignore

@@ -4,3 +4,5 @@ target
 *.sublime*
 *~
 *.pyc
+.vscode
+*.core

AGENTS.md

@@ -0,0 +1,19 @@
+# electrs
+
+## Rules
+
+1. You are an expert Rust developer.
+2. You are an expert Bitcoin developer.
+3. If you are unsure of a change, ask the developer to make a choice proactively.
+
+## Before testing
+
+- Run cargo fmt (from root)
+  - command: `cargo fmt`
+
+## Testing
+
+- Run the checks script
+  - `./scripts/checks.sh`
+- Run with tests only when a test is added or changed
+  - `INCLUDE_TESTS=1 ./scripts/checks.sh`

Cargo.toml

@@ -1,70 +1,69 @@
 [package]
-name = "electrs"
-version = "0.4.1"
-authors = ["Roman Zeyde <me@romanzey.de>"]
+name = "mempool-electrs"
+version = "3.4.0-dev"
+authors = [
+    "Roman Zeyde <me@romanzey.de>",
+    "Nadav Ivgi <nadav@shesek.info>",
+    "wiz <j@wiz.biz>",
+    "junderw <jonathan.underwood4649@gmail.com>"
+]
 description = "An efficient re-implementation of Electrum Server in Rust"
 license = "MIT"
-homepage = "https://github.com/romanz/electrs"
-repository = "https://github.com/romanz/electrs"
+homepage = "https://github.com/mempool/electrs"
+repository = "https://github.com/mempool/electrs"
+publish = false
 keywords = ["bitcoin", "electrum", "server", "index", "database"]
-documentation = "https://docs.rs/electrs/"
 readme = "README.md"
 edition = "2018"
 
+[lib]
+name = "electrs"
+
 [features]
-default = [ "rocksdb" ]
-liquid = [ "elements" ]
-electrum-discovery = [ "electrum-client"]
-oldcpu = [ "rocksdb-oldcpu" ]
+default = []
+liquid = ["elements"]
+electrum-discovery = ["electrum-client"]
 
 [dependencies]
-arraydeque = "0.4"
 arrayref = "0.3.6"
-base64 = "0.12.3"
-bincode = "1.3.1"
-clap = "2.31"
-crossbeam-channel = "0.4"
-dirs = "3.0.1"
-error-chain = "0.12.2"
+base64 = "0.13.0"
+bincode-do-not-use-directly = { version = "1.3.1", package = "bincode" }
+bitcoin = { version = "0.32.8", features = [ "serde" ] }
+bounded-vec-deque = "0.1.1"
+clap = "2.33.3"
+crossbeam-channel = "0.5.0"
+dirs = "4.0.0"
+elements = { version = "0.26.1", features = [ "serde" ], optional = true }
+error-chain = "0.12.4"
 glob = "0.3"
 hex = "0.4.2"
-itertools = "0.9.0"
+itertools = "0.10"
 lazy_static = "1.3.0"
-libc = "0.2.68"
-log = "0.4"
-socket2 = { version = "0.3.12", features = ["reuseport"] }
+libc = "0.2"
+log = "0.4.11"
+socket2 = { version = "0.4", features = ["all"] }
 num_cpus = "1.12.0"
 page_size = "0.4.2"
-prometheus = "0.9"
-rayon = "1.3.1"
-rocksdb = { version = "0.14.0", optional = true }
-rocksdb-oldcpu = { version = "0.12.4", optional = true, package = "rocksdb" }
-rust-crypto = "0.2"
-serde = "1.0.114"
-serde_derive = "1.0.114"
-serde_json = "1.0.56"
-signal-hook = "0.1"
-stderrlog = "0.4.3"
-sysconf = ">=0.3.4"
-time = "0.2.16"
-tiny_http = "0.7.0"
-url = "2.1.1"
-hyper = "0.13.6"
-hyperlocal = "0.7"
-# close to same tokio version as dependent by hyper v0.13.6 and hyperlocal 0.7 -- things can go awry if they mismatch
-tokio = { version = "=0.2.6", features = ["sync", "macros"] }
+prometheus = "0.13"
+ppp = "2.3.0"
+rayon = "1.5.0"
+rocksdb = "0.24.0"
+serde = "1.0.118"
+serde_derive = "1.0.118"
+serde_json = "1.0.60"
+sha2 = "0.10.7"
+signal-hook = "0.3"
+stderrlog = "0.5.0"
+time = { version = "0.3", features = ["formatting"] }
+tiny_http = "0.11"
+url = "2.2.0"
+hyper = "0.14"
+hyperlocal = "0.8"
+# close to same tokio version as dependent by hyper v0.14 and hyperlocal 0.8 -- things can go awry if they mismatch
+tokio = { version = "1", features = ["sync", "macros"] }
 
 # optional dependencies for electrum-discovery
-electrum-client = { version = "0.1.0-beta.6", optional = true }
-
-[dependencies.bitcoin]
-version = "0.23.0"
-features = ["use-serde"]
-
-[dependencies.elements]
-optional = true
-version = "0.12.1"
-features = ["serde-feature"]
+electrum-client = { version = "0.24.1", optional = true }
 
 [dev-dependencies]
 tempfile = "3.0"
@@ -74,14 +73,6 @@ lto = true
 panic = 'abort'
 codegen-units = 1
 
-[patch.crates-io.bitcoin]
-git = "https://github.com/blockstream/rust-bitcoin"
-rev = "0.23-electrs"
-
-[patch.crates-io.elements]
-git = "https://github.com/elementsproject/rust-elements"
-rev = "98e0c4ba1b55cd876d1d8bcff64bcf01c4c370b6"
-
 [patch.crates-io.electrum-client]
-git = "https://github.com/Blockstream/rust-electrum-client"
-rev = "bd783aa70604fa1a0d8b29a3459326c640001354" # add-peer
+git = "https://github.com/mempool/rust-electrum-client"
+rev = "4bbfc612d594fe23282c439d4bdc446cff01ba1c" # 0.24.1/add-peer branch

Dockerfile

@@ -1,21 +1,30 @@
-FROM rust:latest
+FROM debian:bookworm-slim AS base
 
-RUN apt-get update
-RUN apt-get install -y clang cmake
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
 
-RUN cargo install electrs
+RUN apt update -qy && \
+    apt install -qy librocksdb-dev curl
 
-RUN adduser --disabled-login --system --shell /bin/false --uid 1000 user
+FROM base as build
 
-USER user
-WORKDIR /home/user
+RUN apt install -qy git clang cmake
+
+ENV RUSTUP_HOME=/rust
+ENV CARGO_HOME=/cargo 
+ENV PATH=/cargo/bin:/rust/bin:$PATH
+
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+WORKDIR /build
+COPY . .
+
+ARG CARGO_EXTRA_ARGS=""
+RUN cargo build --release --bin electrs ${CARGO_EXTRA_ARGS}
+
+FROM base as deploy
+
+COPY --from=build /build/target/release/electrs /bin/electrs
 
-# Electrum RPC
 EXPOSE 50001
 
-# Prometheus monitoring
-EXPOSE 4224
-
-STOPSIGNAL SIGINT
-
-CMD ["electrs", "-vvvv", "--timestamp"]
+ENTRYPOINT ["/bin/electrs"]

README.md

@@ -1,10 +1,10 @@
-# Esplora - Electrs backend API
+# Mempool - Electrs backend API
 
-A block chain index engine and HTTP API written in Rust based on [romanz/electrs](https://github.com/romanz/electrs).
+A block chain index engine and HTTP API written in Rust based on [romanz/electrs](https://github.com/romanz/electrs) and [Blockstream/electrs](https://github.com/Blockstream/electrs).
 
-Used as the backend for the [Esplora block explorer](https://github.com/Blockstream/esplora) powering [blockstream.info](https://blockstream.info/).
+Used as the backend for the [mempool block explorer](https://github.com/mempool/mempool) powering [mempool.space](https://mempool.space/).
 
-API documentation [is available here](https://github.com/blockstream/esplora/blob/master/API.md).
+API documentation [is available here](https://mempool.space/docs/api/rest).
 
 Documentation for the database schema and indexing process [is available here](doc/schema.md).
 
@@ -13,8 +13,8 @@ Documentation for the database schema and indexing process [is available here](d
 Install Rust, Bitcoin Core (no `txindex` needed) and the `clang` and `cmake` packages, then:
 
 ```bash
-$ git clone https://github.com/blockstream/electrs && cd electrs
-$ git checkout new-index
+$ git clone https://github.com/mempool/electrs && cd electrs
+$ git checkout mempool
 $ cargo run --release --bin electrs -- -vvvv --daemon-dir ~/.bitcoin
 
 # Or for liquid:
@@ -24,11 +24,9 @@ $ cargo run --features liquid --release --bin electrs -- -vvvv --network liquid
 See [electrs's original documentation](https://github.com/romanz/electrs/blob/master/doc/usage.md) for more detailed instructions.
 Note that our indexes are incompatible with electrs's and has to be created separately.
 
-The indexes require 610GB of storage after running compaction (as of June 2020), but you'll need to have
+The indexes require 1.3TB of storage after running compaction (as of October 2023), but you'll need to have
 free space of about double that available during the index compaction process.
-Creating the indexes should take a few hours on a beefy machine with SSD.
-
-To deploy with Docker, follow the [instructions here](https://github.com/Blockstream/esplora#how-to-build-the-docker-image).
+Creating the indexes should take a few hours on a beefy machine with high speed NVMe SSD(s).
 
 ### Light mode
 
@@ -66,6 +64,7 @@ In addition to electrs's original configuration options, a few new options are a
 - `--lightmode` - enable light mode (see above)
 - `--cors <origins>` - origins allowed to make cross-site request (optional, defaults to none).
 - `--address-search` - enables the by-prefix address search index.
+- `--index-unspendables` - enables indexing of provably unspendable outputs.
 - `--utxos-limit <num>` - maximum number of utxos to return per address.
 - `--electrum-txs-limit <num>` - maximum number of txs to return per address in the electrum server (does not apply for the http api).
 - `--electrum-banner <text>` - welcome banner text for electrum server.
@@ -77,7 +76,7 @@ Additional options with the `electrum-discovery` feature:
 - `--electrum-hosts <json>` - a json map of the public hosts where the electrum server is reachable, in the [`server.features` format](https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server.features).
 - `--electrum-announce` - announce the electrum server on the electrum p2p server discovery network.
 
-See `$ cargo run --release --bin electrs -- --help` for the full list of options.
+See `$ cargo run --bin electrs -- --help` for the full list of options.
 
 ## License
 

build.rs

@@ -0,0 +1,57 @@
+use std::{path::Path, process::Command};
+
+fn main() {
+    // Specify re-run conditions
+
+    // 1. Rerun build script if we pass a new GIT_HASH
+    println!("cargo:rerun-if-env-changed=GIT_HASH");
+
+    // 2. Only do git based reruns if git directory exists
+    if Path::new(".git").exists() {
+        // If we change the branch, rerun
+        println!("cargo:rerun-if-changed=.git/HEAD");
+        if let Ok(r) = std::fs::read_to_string(".git/HEAD") {
+            if let Some(stripped) = r.strip_prefix("ref: ") {
+                // If the HEAD is detached it will be a commit hash
+                // so the HEAD changed directive above will pick it up,
+                // otherwise it will point to a ref in the refs directory
+                println!("cargo:rerun-if-changed=.git/{}", stripped);
+            }
+        }
+    }
+
+    // Getting git hash
+
+    // Don't fetch git hash if it's already in the ENV
+    let existing = std::env::var("GIT_HASH").unwrap_or_else(|_| String::new());
+    if !existing.is_empty() {
+        return;
+    }
+
+    // Get git hash from git and don't do anything if the command fails
+    if let Some(rev_parse) = cmd("git", &["rev-parse", "--short", "HEAD"]) {
+        // Add (dirty) to the GIT_HASH if the git status isn't clean
+        // This includes untracked files
+        let dirty = cmd("git", &["status", "--short"]).expect("git command works");
+
+        // Ignore Dockerfile deletion as it is expected in Docker buildx builds
+        let git_hash = if dirty.is_empty() || dirty.trim() == "D Dockerfile" {
+            rev_parse
+        } else {
+            format!("{}(dirty)", rev_parse.trim())
+        };
+
+        println!("cargo:rustc-env=GIT_HASH={}", git_hash.trim());
+    }
+}
+
+// Helper function, Command is verbose...
+fn cmd(name: &str, args: &[&str]) -> Option<String> {
+    Command::new(name).args(args).output().ok().and_then(|o| {
+        if o.status.success() {
+            String::from_utf8(o.stdout).ok()
+        } else {
+            None
+        }
+    })
+}

doc/schema.md

@@ -43,7 +43,7 @@ When the indexer is synced up to the tip of the chain, the hash of the tip is sa
 
 ### `history`
 
-Each funding output (except for provably unspendable ones) results in the following new rows (`H` is for history, `F` is for funding):
+Each funding output (except for provably unspendable ones when `--index-unspendables` is not enabled) results in the following new rows (`H` is for history, `F` is for funding):
 
  * `"H{funding-scripthash}{funding-height}F{funding-txid:vout}{value}" → ""`
  * `"a{funding-address-str}" → ""` (for prefix address search, only saved when `--address-search` is enabled)

docker-compose.yml

@@ -0,0 +1,24 @@
+version: '3.9'
+services:
+
+  mempool-electrs:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: on-failure
+
+    ports:
+      - 50001:50001
+
+    entrypoint:
+      /bin/electrs
+    command: |
+      --address-search
+      --cookie mempool:mempool
+      --db-dir /electrs
+      --cors '*'
+    volumes:
+    - 'electrs_data:/electrs'
+
+volumes:
+  electrs_data:

init-electrs-sockets

@@ -0,0 +1,24 @@
+#!/usr/bin/env zsh
+export ZPOOL=nvm
+
+export BITCOIN_HOME=/bitcoin
+export BITCOIN_USER=bitcoin
+export BITCOIN_GROUP=bitcoin
+
+export ELEMENTS_HOME=/elements
+export ELEMENTS_USER=elements
+export ELEMENTS_GROUP=elements
+
+# create /bitcoin/socket with custom ACL for electrs unix sockets
+zfs create -o "mountpoint=${BITCOIN_HOME}/socket" "${ZPOOL}/bitcoin/socket"
+
+# create /elements/socket with custom ACL for electrs unix sockets
+zfs create -o "mountpoint=${ELEMENTS_HOME}/socket" "${ZPOOL}/elements/socket"
+
+setfacl -m "user:bitcoin:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${BITCOIN_HOME}/socket"
+
+chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket"
+
+setfacl -m "user:elements:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${ELEMENTS_HOME}/socket"
+
+chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket"

rust-toolchain

@@ -0,0 +1 @@
+1.87

scripts/checks.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -e
+
+# This script is used for running all the checks
+# needed to make sure CI passes.
+
+# See below for the reasoning behind testing
+# clippy for all feature combinations.
+# (Note: clippy reports compile errors too)
+
+# You can pass extra args to cargo commands
+#   ./scripts/check.sh --release
+# will run in release mode. You can use that
+# if you already have release mode compiled, as
+# it will be faster to use the version you already compiled
+
+TESTNAME=""
+cleanup() {
+    exit_code=$?
+    if [[ ${exit_code} -ne 0 ]]; then
+        echo -e "\n\n##### Failed on \"$TESTNAME\"" 1>&2
+    fi
+    exit $exit_code
+}
+trap cleanup EXIT
+
+TESTNAME="Running cargo check"
+echo "$TESTNAME"
+cargo check $@ -q --all-features
+
+TESTNAME="Running cargo fmt check"
+echo "$TESTNAME"
+cargo fmt $@ -q --all -- --check
+
+# Testing all the combinations of clippy.
+# There were many instances where a certain struct
+# differed based on liquid or not(liquid) etc.
+# and the clippy fixes would break the other
+# feature combination.
+#
+# Be prepared to use #[allow(clippy::___)] attributes
+# to "fix" contradictions between feature sets.
+
+TESTNAME="Running cargo clippy check no features"
+echo "$TESTNAME"
+cargo clippy $@ -q
+
+TESTNAME="Running cargo clippy check electrum-discovery"
+echo "$TESTNAME"
+cargo clippy $@ -q -F electrum-discovery
+
+TESTNAME="Running cargo clippy check liquid"
+echo "$TESTNAME"
+cargo clippy $@ -q -F liquid
+
+TESTNAME="Running cargo clippy check electrum-discovery + liquid"
+echo "$TESTNAME"
+cargo clippy $@ -q -F electrum-discovery,liquid
+
+if [ $INCLUDE_TESTS ]; then
+    TESTNAME="Running cargo test with all features"
+    echo "$TESTNAME"
+    cargo test $@ -q --lib --all-features
+fi

src/bin/electrs.rs

@@ -1,4 +1,3 @@
-extern crate bitcoin;
 extern crate error_chain;
 #[macro_use]
 extern crate log;
@@ -46,11 +45,12 @@ fn run_server(config: Arc<Config>) -> Result<()> {
     metrics.start();
 
     let daemon = Arc::new(Daemon::new(
-        &config.daemon_dir,
-        &config.blocks_dir,
+        config.daemon_dir.clone(),
+        config.blocks_dir.clone(),
         config.daemon_rpc_addr,
         config.cookie_getter(),
         config.network_type,
+        config.magic,
         signal.clone(),
         &metrics,
     )?);
@@ -70,24 +70,30 @@ fn run_server(config: Arc<Config>) -> Result<()> {
         &metrics,
     ));
 
-    if let Some(ref precache_file) = config.precache_scripts {
-        let precache_scripthashes = precache::scripthashes_from_file(precache_file.to_string())
-            .expect("cannot load scripts to precache");
-        precache::precache(&chain, precache_scripthashes);
-    }
-
     let mempool = Arc::new(RwLock::new(Mempool::new(
         Arc::clone(&chain),
         &metrics,
         Arc::clone(&config),
     )));
-    mempool.write().unwrap().update(&daemon)?;
+    loop {
+        match Mempool::update(&mempool, &daemon) {
+            Ok(_) => break,
+            Err(e) => {
+                warn!(
+                    "Error performing initial mempool update, trying again in 5 seconds: {}",
+                    e.display_chain()
+                );
+                signal.wait(Duration::from_secs(5), false)?;
+            }
+        }
+    }
 
     #[cfg(feature = "liquid")]
-    let asset_db = config
-        .asset_db_path
-        .as_ref()
-        .map(|dir| AssetRegistry::new(dir.clone()));
+    let asset_db = config.asset_db_path.as_ref().map(|db_dir| {
+        let asset_db = Arc::new(RwLock::new(AssetRegistry::new(db_dir.clone())));
+        AssetRegistry::spawn_sync(asset_db.clone());
+        asset_db
+    });
 
     let query = Arc::new(Query::new(
         Arc::clone(&chain),
@@ -99,12 +105,36 @@ fn run_server(config: Arc<Config>) -> Result<()> {
     ));
 
     // TODO: configuration for which servers to start
-    let rest_server = rest::start(Arc::clone(&config), Arc::clone(&query));
+    let rest_server = rest::start(Arc::clone(&config), Arc::clone(&query), &metrics);
     let electrum_server = ElectrumRPC::start(Arc::clone(&config), Arc::clone(&query), &metrics);
 
+    if let Some(ref precache_file) = config.precache_scripts {
+        let precache_scripthashes = precache::scripthashes_from_file(precache_file.to_string())
+            .expect("cannot load scripts to precache");
+        precache::precache(
+            Arc::clone(&chain),
+            precache_scripthashes,
+            config.precache_threads,
+        );
+    }
+
     loop {
-        if let Err(err) = signal.wait(Duration::from_secs(5), true) {
+        if let Err(err) = signal.wait(Duration::from_millis(config.main_loop_delay), true) {
             info!("stopping server: {}", err);
+
+            electrs::util::spawn_thread("shutdown-thread-checker", || {
+                let mut counter = 40;
+                let interval_ms = 500;
+
+                while counter > 0 {
+                    electrs::util::with_spawned_threads(|threads| {
+                        debug!("Threads during shutdown: {:?}", threads);
+                    });
+                    std::thread::sleep(std::time::Duration::from_millis(interval_ms));
+                    counter -= 1;
+                }
+            });
+
             rest_server.stop();
             // the electrum server is stopped when dropped
             break;
@@ -118,7 +148,13 @@ fn run_server(config: Arc<Config>) -> Result<()> {
         };
 
         // Update mempool
-        mempool.write().unwrap().update(&daemon)?;
+        if let Err(e) = Mempool::update(&mempool, &daemon) {
+            // Log the error if the result is an Err
+            warn!(
+                "Error updating mempool, skipping mempool update: {}",
+                e.display_chain()
+            );
+        }
 
         // Update subscribed clients
         electrum_server.notify();
@@ -133,4 +169,7 @@ fn main() {
         error!("server failed: {}", e.display_chain());
         process::exit(1);
     }
+    electrs::util::with_spawned_threads(|threads| {
+        debug!("Threads before closing: {:?}", threads);
+    });
 }

src/bin/popular-scripts.rs

@@ -1,52 +1,183 @@
 extern crate electrs;
 
-use bincode::Options;
-use electrs::{
-    config::Config,
-    new_index::{Store, TxHistoryKey},
-};
+use std::{convert::TryInto, thread::ThreadId, time::Instant};
 
+use electrs::{config::Config, new_index::db::open_raw_db};
+use lazy_static::lazy_static;
+
+/*
+// How to run:
+export ELECTRS_DATA=/path/to/electrs
+cargo run \
+  -q --release --bin popular-scripts -- \
+  --db-dir $ELECTRS_DATA/db \
+  > ./contrib/popular-scripts.txt
+*/
+
+type DB = rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>;
+lazy_static! {
+    static ref HISTORY_DB: DB = {
+        let config = Config::from_args();
+        open_raw_db(
+            &config.db_path.join("newindex").join("history"),
+            electrs::new_index::db::OpenMode::ReadOnly,
+        )
+    };
+}
+
+// Dev note:
+// Only use println for file output (lines for output)
+// Use eprintln to print to stderr for dev notifications
 fn main() {
-    let config = Config::from_args();
-    let store = Store::open(&config.db_path.join("newindex"), &config);
+    let high_usage_threshold = std::env::var("HIGH_USAGE_THRESHOLD")
+        .ok()
+        .and_then(|s| s.parse::<u32>().ok())
+        .unwrap_or(4000);
+    let thread_count = std::env::var("JOB_THREAD_COUNT")
+        .ok()
+        .and_then(|s| s.parse::<usize>().ok())
+        .unwrap_or(4);
+    eprintln!(
+        "Seaching for scripts with history rows of {} or more...",
+        high_usage_threshold
+    );
 
-    let mut iter = store.history_db().raw_iterator();
-    iter.seek(b"H");
+    let thread_pool = rayon::ThreadPoolBuilder::new()
+        .num_threads(thread_count)
+        .build()
+        .expect("Built threadpool");
+
+    let (sender, receiver) = crossbeam_channel::unbounded::<[u8; 32]>();
+
+    let increment = 256 / thread_count;
+    let bytes: Vec<u8> = (0u8..=255u8)
+        .filter(|n| *n % increment as u8 == 0)
+        .collect();
+
+    let now = Instant::now();
+    for i in 0..bytes.len() {
+        let sender = sender.clone();
+        let first_byte = bytes[i];
+        let second_byte = bytes.get(i + 1).copied();
+
+        thread_pool.spawn(move || {
+            let id = std::thread::current().id();
+            run_iterator(
+                id,
+                &HISTORY_DB,
+                high_usage_threshold,
+                first_byte,
+                second_byte,
+                sender,
+                now,
+            );
+            eprintln!("{id:?} Finished its job!");
+        })
+    }
+    // If we don't drop this sender
+    // the receiver will hang forever
+    drop(sender);
+
+    while let Ok(script) = receiver.recv() {
+        println!("{}", hex::encode(script));
+    }
+    eprintln!("Finished!!!!");
+}
+
+fn run_iterator(
+    thread_id: ThreadId,
+    db: &DB,
+    high_usage_threshold: u32,
+    first_byte: u8,
+    next_byte: Option<u8>,
+    sender: crossbeam_channel::Sender<[u8; 32]>,
+    now: Instant,
+) {
+    let mut iter = db.raw_iterator();
+    eprintln!(
+        "Thread ({thread_id:?}) Seeking DB to beginning of tx histories for b'H' + {}",
+        hex::encode([first_byte])
+    );
+    let mut compare_vec: Vec<u8> = vec![b'H', first_byte];
+    iter.seek(&compare_vec); // Seek to beginning of our section
+
+    // Insert the byte of the next section for comparing
+    // This will tell us when to stop with a closure
+    type Checker<'a> = Box<dyn Fn(&[u8]) -> bool + 'a>;
+    let is_finished: Checker<'_> = if let Some(next) = next_byte {
+        // Modify the vec to what we're looking for next
+        // to indicate we left our section
+        compare_vec[1] = next;
+        Box::new(|key: &[u8]| -> bool { key.starts_with(&compare_vec) })
+    } else {
+        // Modify the vec to only have H so we know when we left H
+        compare_vec.remove(1);
+        Box::new(|key: &[u8]| -> bool { !key.starts_with(&compare_vec) })
+    };
+
+    eprintln!("Thread ({thread_id:?}) Seeking done");
 
     let mut curr_scripthash = [0u8; 32];
-    let mut total_entries = 0;
+    let mut total_entries: usize = 0;
+    let mut iter_index: usize = 1;
 
     while iter.valid() {
         let key = iter.key().unwrap();
 
-        if !key.starts_with(b"H") {
+        if key.is_empty() || key[0] != b'H' || is_finished(key) {
+            // We have left the txhistory section,
+            // but we need to check the final scripthash
+            send_if_popular(
+                high_usage_threshold,
+                total_entries,
+                curr_scripthash,
+                &sender,
+            );
             break;
         }
 
-        let entry: TxHistoryKey = bincode::options()
-            .with_big_endian()
-            .deserialize(&key)
-            .expect("failed to deserialize TxHistoryKey");
+        if iter_index % 10_000_000 == 0 {
+            let duration = now.elapsed().as_secs();
+            eprintln!(
+                "Thread ({thread_id:?}) Processing row #{iter_index}... {duration} seconds elapsed"
+            );
+        }
 
-        if curr_scripthash != entry.hash {
-            if total_entries > 100 {
-                println!("{} {}", hex::encode(&curr_scripthash), total_entries);
-            }
+        // We know that the TxHistory key is 1 byte "H" followed by
+        // 32 byte scripthash
+        let entry_hash: [u8; 32] = key[1..33].try_into().unwrap();
 
-            curr_scripthash = entry.hash;
+        if curr_scripthash != entry_hash {
+            // We have rolled on to a new scripthash
+            // If the last scripthash was popular
+            // Collect for sorting
+            send_if_popular(
+                high_usage_threshold,
+                total_entries,
+                curr_scripthash,
+                &sender,
+            );
+
+            // After collecting, reset values for next scripthash
+            curr_scripthash = entry_hash;
             total_entries = 0;
         }
 
         total_entries += 1;
+        iter_index += 1;
 
         iter.next();
     }
+}
 
-    if total_entries >= 4000 {
-        println!(
-            "scripthash,{},{}",
-            hex::encode(&curr_scripthash),
-            total_entries
-        );
+#[inline]
+fn send_if_popular(
+    high_usage_threshold: u32,
+    total_entries: usize,
+    curr_scripthash: [u8; 32],
+    sender: &crossbeam_channel::Sender<[u8; 32]>,
+) {
+    if total_entries >= high_usage_threshold as usize {
+        sender.send(curr_scripthash).unwrap();
     }
 }

src/bin/tx-fingerprint-stats.rs

@@ -9,7 +9,7 @@ fn main() {
     use std::collections::HashSet;
     use std::sync::Arc;
 
-    use bitcoin::blockdata::script::Script;
+    use bitcoin::blockdata::script::ScriptBuf;
     use bitcoin::consensus::encode::deserialize;
     use electrs::{
         chain::Transaction,
@@ -30,11 +30,12 @@ fn main() {
 
     let daemon = Arc::new(
         Daemon::new(
-            &config.daemon_dir,
-            &config.blocks_dir,
+            config.daemon_dir.clone(),
+            config.blocks_dir.clone(),
             config.daemon_rpc_addr,
             config.cookie_getter(),
             config.network_type,
+            config.magic,
             signal,
             &metrics,
         )
@@ -60,8 +61,8 @@ fn main() {
             break;
         }
 
-        let tx: Transaction = deserialize(&value).expect("failed to parse Transaction");
-        let txid = tx.txid();
+        let tx: Transaction = deserialize(value).expect("failed to parse Transaction");
+        let txid = tx.compute_txid();
 
         iter.next();
 
@@ -70,7 +71,7 @@ fn main() {
             continue;
         }
         // skip coinbase txs
-        if tx.is_coin_base() {
+        if tx.is_coinbase() {
             continue;
         }
 
@@ -90,12 +91,26 @@ fn main() {
                 .collect(),
         );
 
-        let total_out: u64 = tx.output.iter().map(|out| out.value).sum();
-        let small_out = tx.output.iter().map(|out| out.value).min().unwrap();
-        let large_out = tx.output.iter().map(|out| out.value).max().unwrap();
+        let total_out: u64 = tx.output.iter().map(|out| out.value.to_sat()).sum();
+        let small_out = tx
+            .output
+            .iter()
+            .map(|out| out.value.to_sat())
+            .min()
+            .unwrap();
+        let large_out = tx
+            .output
+            .iter()
+            .map(|out| out.value.to_sat())
+            .max()
+            .unwrap();
 
-        let total_in: u64 = prevouts.values().map(|out| out.value).sum();
-        let smallest_in = prevouts.values().map(|out| out.value).min().unwrap();
+        let total_in: u64 = prevouts.values().map(|out| out.value.to_sat()).sum();
+        let smallest_in = prevouts
+            .values()
+            .map(|out| out.value.to_sat())
+            .min()
+            .unwrap();
 
         let fee = total_in - total_out;
 
@@ -118,7 +133,7 @@ fn main() {
 
         // test for sending back to one of the spent spks
         let has_reuse = {
-            let prev_spks: HashSet<Script> = prevouts
+            let prev_spks: HashSet<ScriptBuf> = prevouts
                 .values()
                 .map(|out| out.script_pubkey.clone())
                 .collect();

src/chain.rs

@@ -1,66 +1,132 @@
+use std::str::FromStr;
+
 #[cfg(not(feature = "liquid"))] // use regular Bitcoin data structures
-pub use bitcoin::{util::address, Block, BlockHeader, OutPoint, Transaction, TxIn, TxOut};
+pub use bitcoin::{
+    address,
+    block::Header as BlockHeader,
+    blockdata::{opcodes, script},
+    consensus::deserialize,
+    hashes, Block, BlockHash, OutPoint, ScriptBuf as Script, Transaction, TxIn, TxOut, Txid,
+    Witness,
+};
 
 #[cfg(feature = "liquid")]
 pub use {
     crate::elements::asset,
     elements::{
-        address, confidential, Address, AssetId, Block, BlockHeader, OutPoint, Transaction, TxIn,
-        TxOut,
+        address, bitcoin::bech32::Hrp, confidential, encode::deserialize, hashes, opcodes, script,
+        Address, AssetId, Block, BlockHash, BlockHeader, OutPoint, Script, Transaction, TxIn,
+        TxInWitness as Witness, TxOut, Txid,
     },
 };
 
 use bitcoin::blockdata::constants::genesis_block;
-use bitcoin::network::constants::Network as BNetwork;
-use bitcoin::util::hash::BitcoinHash;
-use bitcoin::BlockHash;
+pub use bitcoin::Network as BNetwork;
 
-use std::collections::HashMap;
-use std::sync::{Arc, RwLock};
+// Extension trait for getting txid in a cross-compatible way
+pub trait TxidCompat {
+    fn get_txid(&self) -> Txid;
+}
+
+#[cfg(not(feature = "liquid"))]
+impl TxidCompat for Transaction {
+    fn get_txid(&self) -> Txid {
+        self.compute_txid()
+    }
+}
+
+#[cfg(feature = "liquid")]
+impl TxidCompat for Transaction {
+    fn get_txid(&self) -> Txid {
+        self.txid()
+    }
+}
+
+// Extension trait for getting block size in a cross-compatible way
+pub trait BlockSizeCompat {
+    fn get_block_size(&self) -> usize;
+}
+
+#[cfg(not(feature = "liquid"))]
+impl BlockSizeCompat for Block {
+    fn get_block_size(&self) -> usize {
+        self.total_size()
+    }
+}
+
+#[cfg(feature = "liquid")]
+impl BlockSizeCompat for Block {
+    fn get_block_size(&self) -> usize {
+        self.size()
+    }
+}
 
 #[cfg(not(feature = "liquid"))]
 pub type Value = u64;
 #[cfg(feature = "liquid")]
 pub use confidential::Value;
 
-lazy_static! {
-    static ref CACHED_GENESIS: Arc<RwLock<HashMap<Network, BlockHash>>> =
-        Arc::new(RwLock::new(HashMap::new()));
-}
-
 #[derive(Debug, Copy, Clone, PartialEq, Hash, Serialize, Ord, PartialOrd, Eq)]
 pub enum Network {
+    #[cfg(not(feature = "liquid"))]
     Bitcoin,
+    #[cfg(not(feature = "liquid"))]
     Testnet,
+    #[cfg(not(feature = "liquid"))]
+    Testnet4,
+    #[cfg(not(feature = "liquid"))]
     Regtest,
+    #[cfg(not(feature = "liquid"))]
+    Signet,
 
     #[cfg(feature = "liquid")]
     Liquid,
     #[cfg(feature = "liquid")]
+    LiquidTestnet,
+    #[cfg(feature = "liquid")]
     LiquidRegtest,
 }
 
+#[cfg(feature = "liquid")]
+pub const LIQUID_TESTNET_PARAMS: address::AddressParams = address::AddressParams {
+    p2pkh_prefix: 36,
+    p2sh_prefix: 19,
+    blinded_prefix: 23,
+    bech_hrp: Hrp::parse_unchecked("tex"),
+    blech_hrp: Hrp::parse_unchecked("tlq"),
+};
+
+/// Magic for testnet4, 0x1c163f28 (from BIP94) with flipped endianness.
+#[cfg(not(feature = "liquid"))]
+const TESTNET4_MAGIC: u32 = 0x283f161c;
+
 impl Network {
-    pub fn genesis_hash(self) -> BlockHash {
-        if let Some(block_hash) = CACHED_GENESIS.read().unwrap().get(&self) {
-            return *block_hash;
-        }
-
-        let block_hash = genesis_block(BNetwork::from(self)).bitcoin_hash();
-        CACHED_GENESIS.write().unwrap().insert(self, block_hash);
-        block_hash
-    }
-
+    #[cfg(not(feature = "liquid"))]
     pub fn magic(self) -> u32 {
         match self {
-            Network::Bitcoin => 0xD9B4_BEF9,
-            Network::Testnet => 0x0709_110B,
-            Network::Regtest => 0xDAB5_BFFA,
+            Self::Testnet4 => TESTNET4_MAGIC,
+            _ => {
+                let magic = BNetwork::from(self).magic();
+                u32::from_le_bytes(magic.to_bytes())
+            }
+        }
+    }
 
+    #[cfg(feature = "liquid")]
+    pub fn magic(self) -> u32 {
+        match self {
+            Network::Liquid | Network::LiquidRegtest => 0xDAB5_BFFA,
+            Network::LiquidTestnet => 0x62DD_0E41,
+        }
+    }
+
+    pub fn is_regtest(self) -> bool {
+        match self {
+            #[cfg(not(feature = "liquid"))]
+            Network::Regtest => true,
             #[cfg(feature = "liquid")]
-            Network::Liquid => 0xDAB5_BFFA,
-            #[cfg(feature = "liquid")]
-            Network::LiquidRegtest => 0xDAB5_BFFA,
+            Network::LiquidRegtest => true,
+            _ => false,
         }
     }
 
@@ -70,17 +136,24 @@ impl Network {
         match self {
             Network::Liquid => &address::AddressParams::LIQUID,
             Network::LiquidRegtest => &address::AddressParams::ELEMENTS,
-            _ => panic!("the liquid-only address_params() called with non-liquid network"),
+            Network::LiquidTestnet => &LIQUID_TESTNET_PARAMS,
         }
     }
 
     #[cfg(feature = "liquid")]
     pub fn native_asset(self) -> &'static AssetId {
         match self {
-            Network::Liquid => &*asset::NATIVE_ASSET_ID,
-            // same for testnet and regtest
-            Network::LiquidRegtest => &*asset::NATIVE_ASSET_ID_TESTNET,
-            _ => panic!("the liquid-only native_asset_id() called with non-liquid network"),
+            Network::Liquid => &asset::NATIVE_ASSET_ID,
+            Network::LiquidTestnet => &asset::NATIVE_ASSET_ID_TESTNET,
+            Network::LiquidRegtest => &asset::NATIVE_ASSET_ID_REGTEST,
+        }
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn pegged_asset(self) -> Option<&'static AssetId> {
+        match self {
+            Network::Liquid => Some(&*asset::NATIVE_ASSET_ID),
+            Network::LiquidTestnet | Network::LiquidRegtest => None,
         }
     }
 
@@ -90,29 +163,97 @@ impl Network {
             "mainnet".to_string(),
             "testnet".to_string(),
             "regtest".to_string(),
+            "signet".to_string(),
         ];
 
         #[cfg(feature = "liquid")]
         return vec![
-            "mainnet".to_string(),
-            "testnet".to_string(),
-            "regtest".to_string(),
             "liquid".to_string(),
+            "liquidtestnet".to_string(),
             "liquidregtest".to_string(),
         ];
     }
 }
 
+pub fn genesis_hash(network: Network) -> BlockHash {
+    #[cfg(not(feature = "liquid"))]
+    return bitcoin_genesis_hash(network);
+    #[cfg(feature = "liquid")]
+    return liquid_genesis_hash(network);
+}
+
+pub fn bitcoin_genesis_hash(network: Network) -> bitcoin::BlockHash {
+    lazy_static! {
+        static ref BITCOIN_GENESIS: bitcoin::BlockHash =
+            genesis_block(BNetwork::Bitcoin).block_hash();
+        static ref TESTNET_GENESIS: bitcoin::BlockHash =
+            genesis_block(BNetwork::Testnet).block_hash();
+        static ref TESTNET4_GENESIS: bitcoin::BlockHash = bitcoin::BlockHash::from_str(
+            "00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043"
+        )
+        .unwrap();
+        static ref REGTEST_GENESIS: bitcoin::BlockHash =
+            genesis_block(BNetwork::Regtest).block_hash();
+        static ref SIGNET_GENESIS: bitcoin::BlockHash =
+            genesis_block(BNetwork::Signet).block_hash();
+    }
+    #[cfg(not(feature = "liquid"))]
+    match network {
+        Network::Bitcoin => *BITCOIN_GENESIS,
+        Network::Testnet => *TESTNET_GENESIS,
+        Network::Testnet4 => *TESTNET4_GENESIS,
+        Network::Regtest => *REGTEST_GENESIS,
+        Network::Signet => *SIGNET_GENESIS,
+    }
+    #[cfg(feature = "liquid")]
+    match network {
+        Network::Liquid => *BITCOIN_GENESIS,
+        Network::LiquidTestnet => *TESTNET_GENESIS,
+        Network::LiquidRegtest => *REGTEST_GENESIS,
+    }
+}
+
+#[cfg(feature = "liquid")]
+pub fn liquid_genesis_hash(network: Network) -> elements::BlockHash {
+    lazy_static! {
+        static ref LIQUID_GENESIS: BlockHash =
+            "1466275836220db2944ca059a3a10ef6fd2ea684b0688d2c379296888a206003"
+                .parse()
+                .unwrap();
+        static ref ZERO_HASH: BlockHash =
+            "0000000000000000000000000000000000000000000000000000000000000000"
+                .parse()
+                .unwrap();
+    }
+
+    match network {
+        Network::Liquid => *LIQUID_GENESIS,
+        // The genesis block for liquid regtest chains varies based on the chain configuration.
+        // This instead uses an all zeroed-out hash, which doesn't matter in practice because its
+        // only used for Electrum server discovery, which isn't active on regtest.
+        _ => *ZERO_HASH,
+    }
+}
+
 impl From<&str> for Network {
     fn from(network_name: &str) -> Self {
         match network_name {
+            #[cfg(not(feature = "liquid"))]
             "mainnet" => Network::Bitcoin,
+            #[cfg(not(feature = "liquid"))]
             "testnet" => Network::Testnet,
+            #[cfg(not(feature = "liquid"))]
+            "testnet4" => Network::Testnet4,
+            #[cfg(not(feature = "liquid"))]
             "regtest" => Network::Regtest,
+            #[cfg(not(feature = "liquid"))]
+            "signet" => Network::Signet,
 
             #[cfg(feature = "liquid")]
             "liquid" => Network::Liquid,
             #[cfg(feature = "liquid")]
+            "liquidtestnet" => Network::LiquidTestnet,
+            #[cfg(feature = "liquid")]
             "liquidregtest" => Network::LiquidRegtest,
 
             _ => panic!("unsupported Bitcoin network: {:?}", network_name),
@@ -120,34 +261,28 @@ impl From<&str> for Network {
     }
 }
 
+#[cfg(not(feature = "liquid"))]
 impl From<Network> for BNetwork {
     fn from(network: Network) -> Self {
         match network {
             Network::Bitcoin => BNetwork::Bitcoin,
             Network::Testnet => BNetwork::Testnet,
+            Network::Testnet4 => BNetwork::Testnet4,
             Network::Regtest => BNetwork::Regtest,
-
-            #[cfg(feature = "liquid")]
-            Network::Liquid => BNetwork::Bitcoin, // @FIXME
-            #[cfg(feature = "liquid")]
-            Network::LiquidRegtest => BNetwork::Regtest, // @FIXME
+            Network::Signet => BNetwork::Signet,
         }
     }
 }
 
+#[cfg(not(feature = "liquid"))]
 impl From<BNetwork> for Network {
     fn from(network: BNetwork) -> Self {
         match network {
-            #[cfg(not(feature = "liquid"))]
             BNetwork::Bitcoin => Network::Bitcoin,
-            #[cfg(not(feature = "liquid"))]
+            BNetwork::Testnet => Network::Testnet,
+            BNetwork::Testnet4 => Network::Testnet4,
             BNetwork::Regtest => Network::Regtest,
-
-            #[cfg(feature = "liquid")]
-            BNetwork::Bitcoin => Network::Liquid, // @FIXME
-            #[cfg(feature = "liquid")]
-            BNetwork::Regtest => Network::LiquidRegtest, // @FIXME
-            BNetwork::Testnet => Network::Testnet, // @FIXME
+            BNetwork::Signet => Network::Signet,
         }
     }
 }

src/config.rs

@@ -4,21 +4,38 @@ use std::fs;
 use std::net::SocketAddr;
 use std::net::ToSocketAddrs;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
 use stderrlog;
 
 use crate::chain::Network;
 use crate::daemon::CookieGetter;
-
 use crate::errors::*;
 
-const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION");
+#[cfg(feature = "liquid")]
+use bitcoin::Network as BNetwork;
+
+pub(crate) const APP_NAME: &str = "mempool-electrs";
+pub(crate) const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION");
+pub(crate) const GIT_HASH: Option<&str> = option_env!("GIT_HASH");
+// This will be set only once in the Daemon::new() constructor at startup
+pub(crate) static BITCOIND_SUBVER: OnceLock<String> = OnceLock::new();
+
+lazy_static! {
+    pub(crate) static ref VERSION_STRING: String = {
+        if let Some(hash) = GIT_HASH {
+            format!("{} {}-{}", APP_NAME, ELECTRS_VERSION, hash)
+        } else {
+            format!("{} {}", APP_NAME, ELECTRS_VERSION)
+        }
+    };
+}
 
 #[derive(Debug, Clone)]
 pub struct Config {
     // See below for the documentation of each field:
     pub log: stderrlog::StdErrLog,
     pub network_type: Network,
+    pub magic: Option<u32>,
     pub db_path: PathBuf,
     pub daemon_dir: PathBuf,
     pub blocks_dir: PathBuf,
@@ -27,23 +44,40 @@ pub struct Config {
     pub electrum_rpc_addr: SocketAddr,
     pub http_addr: SocketAddr,
     pub http_socket_file: Option<PathBuf>,
+    pub rpc_socket_file: Option<PathBuf>,
     pub monitoring_addr: SocketAddr,
     pub jsonrpc_import: bool,
     pub light_mode: bool,
+    pub main_loop_delay: u64,
     pub address_search: bool,
+    pub index_unspendables: bool,
     pub cors: Option<String>,
     pub precache_scripts: Option<String>,
+    pub precache_threads: usize,
     pub utxos_limit: usize,
     pub electrum_txs_limit: usize,
     pub electrum_banner: String,
+    pub mempool_backlog_stats_ttl: u64,
+    pub mempool_recent_txs_size: usize,
+    pub rest_default_block_limit: usize,
+    pub rest_default_chain_txs_per_page: usize,
+    pub rest_default_max_mempool_txs: usize,
+    pub rest_default_max_address_summary_txs: usize,
+    pub rest_max_mempool_page_size: usize,
+    pub rest_max_mempool_txid_page_size: usize,
+    pub electrum_max_line_size: usize,
+    pub electrum_max_subscriptions: usize,
+    pub electrum_max_clients: usize,
+    pub electrum_idle_timeout: u64,
+    pub electrum_haproxy_depth: usize,
+    pub electrum_connections_per_client: usize,
+    pub electrum_public_hosts: Option<crate::electrum::ServerHosts>,
 
     #[cfg(feature = "liquid")]
-    pub parent_network: Network,
+    pub parent_network: BNetwork,
     #[cfg(feature = "liquid")]
     pub asset_db_path: Option<PathBuf>,
 
-    #[cfg(feature = "electrum-discovery")]
-    pub electrum_public_hosts: Option<crate::electrum::ServerHosts>,
     #[cfg(feature = "electrum-discovery")]
     pub electrum_announce: bool,
     #[cfg(feature = "electrum-discovery")]
@@ -61,13 +95,15 @@ fn str_to_socketaddr(address: &str, what: &str) -> SocketAddr {
 
 impl Config {
     pub fn from_args() -> Config {
-        let network_help = format!(
-            "Select Bitcoin network type ({})",
-            Network::names().join(", ")
-        );
+        let network_help = format!("Select network type ({})", Network::names().join(", "));
 
-        let args = App::new("Electrum Rust Server")
+        let args = App::new("Mempool Electrum Rust Server")
             .version(crate_version!())
+            .arg(
+                Arg::with_name("version")
+                    .long("version")
+                    .help("Print out the version of this app and quit immediately."),
+            )
             .arg(
                 Arg::with_name("verbosity")
                     .short("v")
@@ -109,6 +145,12 @@ impl Config {
                     .help(&network_help)
                     .takes_value(true),
             )
+            .arg(
+                Arg::with_name("magic")
+                    .long("magic")
+                    .default_value("")
+                    .takes_value(true),
+            )
             .arg(
                 Arg::with_name("electrum_rpc_addr")
                     .long("electrum-rpc-addr")
@@ -143,11 +185,22 @@ impl Config {
                     .long("lightmode")
                     .help("Enable light mode for reduced storage")
             )
+            .arg(
+                Arg::with_name("main_loop_delay")
+                    .long("main-loop-delay")
+                    .help("The number of milliseconds the main loop will wait between loops. (Can be shortened with SIGUSR1)")
+                    .default_value("500")
+            )
             .arg(
                 Arg::with_name("address_search")
                     .long("address-search")
                     .help("Enable prefix address search")
             )
+            .arg(
+                Arg::with_name("index_unspendables")
+                    .long("index-unspendables")
+                    .help("Enable indexing of provably unspendable outputs")
+            )
             .arg(
                 Arg::with_name("cors")
                     .long("cors")
@@ -160,12 +213,66 @@ impl Config {
                     .help("Path to file with list of scripts to pre-cache")
                     .takes_value(true)
             )
+            .arg(
+                Arg::with_name("precache_threads")
+                    .long("precache-threads")
+                    .help("Non-zero number of threads to use for precache threadpool. [default: 4 * CORE_COUNT]")
+                    .takes_value(true)
+            )
             .arg(
                 Arg::with_name("utxos_limit")
                     .long("utxos-limit")
                     .help("Maximum number of utxos to process per address. Lookups for addresses with more utxos will fail. Applies to the Electrum and HTTP APIs.")
                     .default_value("500")
             )
+            .arg(
+                Arg::with_name("mempool_backlog_stats_ttl")
+                    .long("mempool-backlog-stats-ttl")
+                    .help("The number of seconds that need to pass before Mempool::update will update the latency histogram again.")
+                    .default_value("10")
+            )
+            .arg(
+                Arg::with_name("mempool_recent_txs_size")
+                    .long("mempool-recent-txs-size")
+                    .help("The number of transactions that mempool will keep in its recents queue. This is returned by mempool/recent endpoint.")
+                    .default_value("10")
+            )
+            .arg(
+                Arg::with_name("rest_default_block_limit")
+                    .long("rest-default-block-limit")
+                    .help("The default number of blocks returned from the blocks/[start_height] endpoint.")
+                    .default_value("10")
+            )
+            .arg(
+                Arg::with_name("rest_default_chain_txs_per_page")
+                    .long("rest-default-chain-txs-per-page")
+                    .help("The default number of on-chain transactions returned by the txs endpoints.")
+                    .default_value("25")
+            )
+            .arg(
+                Arg::with_name("rest_default_max_mempool_txs")
+                    .long("rest-default-max-mempool-txs")
+                    .help("The default number of mempool transactions returned by the txs endpoints.")
+                    .default_value("50")
+            )
+            .arg(
+                Arg::with_name("rest_default_max_address_summary_txs")
+                    .long("rest-default-max-address-summary-txs")
+                    .help("The default number of transactions returned by the address summary endpoints.")
+                    .default_value("5000")
+            )
+            .arg(
+                Arg::with_name("rest_max_mempool_page_size")
+                    .long("rest-max-mempool-page-size")
+                    .help("The maximum number of transactions returned by the paginated /internal/mempool/txs endpoint.")
+                    .default_value("1000")
+            )
+            .arg(
+                Arg::with_name("rest_max_mempool_txid_page_size")
+                    .long("rest-max-mempool-txid-page-size")
+                    .help("The maximum number of transactions returned by the paginated /mempool/txids/page endpoint.")
+                    .default_value("10000")
+            )
             .arg(
                 Arg::with_name("electrum_txs_limit")
                     .long("electrum-txs-limit")
@@ -176,6 +283,36 @@ impl Config {
                     .long("electrum-banner")
                     .help("Welcome banner for the Electrum server, shown in the console to clients.")
                     .takes_value(true)
+            ).arg(
+                Arg::with_name("electrum_max_line_size")
+                    .long("electrum-max-line-size")
+                    .help("Maximum size of a single Electrum request line in bytes (default: 1 MiB).")
+                    .default_value("1048576")
+            ).arg(
+                Arg::with_name("electrum_max_subscriptions")
+                    .long("electrum-max-subscriptions")
+                    .help("Maximum number of scripthash subscriptions per client connection.")
+                    .default_value("100")
+            ).arg(
+                Arg::with_name("electrum_max_clients")
+                    .long("electrum-max-clients")
+                    .help("Maximum number of concurrent Electrum client connections.")
+                    .default_value("10")
+            ).arg(
+                Arg::with_name("electrum_idle_timeout")
+                    .long("electrum-idle-timeout")
+                    .help("Maximum idle time in seconds since the last client request before disconnecting the Electrum connection.")
+                    .default_value("600")
+            ).arg(
+                Arg::with_name("electrum_haproxy_depth")
+                    .long("electrum-haproxy-depth")
+                    .help("Which HAProxy PROXY-protocol header layer identifies the real client IP. 0 disables PROXY-protocol detection; 1 uses the first (outermost) address, 2 the second, and so on. If the requested layer or any PROXY header is absent, no client IP is associated with the connection.")
+                    .default_value("0")
+            ).arg(
+                Arg::with_name("electrum_connections_per_client")
+                    .long("electrum-connections-per-client")
+                    .help("Maximum number of concurrent Electrum connections allowed per client (keyed by the HAProxy-reported address when available, otherwise the peer IP). 0 disables the per-client limit.")
+                    .default_value("10")
             );
 
         #[cfg(unix)]
@@ -186,6 +323,14 @@ impl Config {
                     .takes_value(true),
             );
 
+        #[cfg(unix)]
+        let args = args.arg(
+                Arg::with_name("rpc_socket_file")
+                    .long("rpc-socket-file")
+                    .help("Electrum RPC 'unix socket file' to listen on (default disabled, enabling this ignores the electrum_rpc_addr arg)")
+                    .takes_value(true),
+            );
+
         #[cfg(feature = "liquid")]
         let args = args
             .arg(
@@ -220,62 +365,105 @@ impl Config {
 
         let m = args.get_matches();
 
+        if m.is_present("version") {
+            eprintln!("{}", *VERSION_STRING);
+            std::process::exit(0);
+        }
+
         let network_name = m.value_of("network").unwrap_or("mainnet");
         let network_type = Network::from(network_name);
+        let magic: Option<u32> = m
+            .value_of("magic")
+            .filter(|s| !s.is_empty())
+            .map(|s| u32::from_str_radix(s, 16).expect("invalid network magic"));
         let db_dir = Path::new(m.value_of("db_dir").unwrap_or("./db"));
         let db_path = db_dir.join(network_name);
 
         #[cfg(feature = "liquid")]
         let parent_network = m
             .value_of("parent_network")
-            .map(Network::from)
+            .map(|s| s.parse().expect("invalid parent network"))
             .unwrap_or_else(|| match network_type {
-                Network::Liquid => Network::Bitcoin,
-                Network::LiquidRegtest => Network::Regtest,
-                _ => panic!("unknown liquid network, --parent-network is required"),
+                Network::Liquid => BNetwork::Bitcoin,
+                // XXX liquid testnet/regtest don't have a parent chain
+                Network::LiquidTestnet | Network::LiquidRegtest => BNetwork::Regtest,
             });
 
         #[cfg(feature = "liquid")]
         let asset_db_path = m.value_of("asset_db_path").map(PathBuf::from);
 
         let default_daemon_port = match network_type {
+            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => 8332,
+            #[cfg(not(feature = "liquid"))]
             Network::Testnet => 18332,
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 18443,
+            #[cfg(not(feature = "liquid"))]
+            Network::Signet => 38332,
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 48332,
 
             #[cfg(feature = "liquid")]
             Network::Liquid => 7041,
             #[cfg(feature = "liquid")]
-            Network::LiquidRegtest => 7041,
+            Network::LiquidTestnet | Network::LiquidRegtest => 7040,
         };
         let default_electrum_port = match network_type {
+            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => 50001,
+            #[cfg(not(feature = "liquid"))]
             Network::Testnet => 60001,
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 40001,
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 60401,
+            #[cfg(not(feature = "liquid"))]
+            Network::Signet => 60601,
 
             #[cfg(feature = "liquid")]
             Network::Liquid => 51000,
             #[cfg(feature = "liquid")]
+            Network::LiquidTestnet => 51301,
+            #[cfg(feature = "liquid")]
             Network::LiquidRegtest => 51401,
         };
         let default_http_port = match network_type {
+            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => 3000,
+            #[cfg(not(feature = "liquid"))]
             Network::Testnet => 3001,
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 3002,
+            #[cfg(not(feature = "liquid"))]
+            Network::Signet => 3003,
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 3004,
 
             #[cfg(feature = "liquid")]
             Network::Liquid => 3000,
             #[cfg(feature = "liquid")]
+            Network::LiquidTestnet => 3001,
+            #[cfg(feature = "liquid")]
             Network::LiquidRegtest => 3002,
         };
         let default_monitoring_port = match network_type {
+            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => 4224,
+            #[cfg(not(feature = "liquid"))]
             Network::Testnet => 14224,
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 24224,
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => 44224,
+            #[cfg(not(feature = "liquid"))]
+            Network::Signet => 54224,
 
             #[cfg(feature = "liquid")]
             Network::Liquid => 34224,
             #[cfg(feature = "liquid")]
+            Network::LiquidTestnet => 44324,
+            #[cfg(feature = "liquid")]
             Network::LiquidRegtest => 44224,
         };
 
@@ -296,6 +484,7 @@ impl Config {
         );
 
         let http_socket_file: Option<PathBuf> = m.value_of("http_socket_file").map(PathBuf::from);
+        let rpc_socket_file: Option<PathBuf> = m.value_of("rpc_socket_file").map(PathBuf::from);
         let monitoring_addr: SocketAddr = str_to_socketaddr(
             m.value_of("monitoring_addr")
                 .unwrap_or(&format!("127.0.0.1:{}", default_monitoring_port)),
@@ -311,13 +500,22 @@ impl Config {
                 default_dir
             });
         match network_type {
+            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => (),
+            #[cfg(not(feature = "liquid"))]
             Network::Testnet => daemon_dir.push("testnet3"),
+            #[cfg(not(feature = "liquid"))]
+            Network::Testnet4 => daemon_dir.push("testnet4"),
+            #[cfg(not(feature = "liquid"))]
             Network::Regtest => daemon_dir.push("regtest"),
+            #[cfg(not(feature = "liquid"))]
+            Network::Signet => daemon_dir.push("signet"),
 
             #[cfg(feature = "liquid")]
             Network::Liquid => daemon_dir.push("liquidv1"),
             #[cfg(feature = "liquid")]
+            Network::LiquidTestnet => daemon_dir.push("liquidtestnet"),
+            #[cfg(feature = "liquid")]
             Network::LiquidRegtest => daemon_dir.push("liquidregtest"),
         }
         let blocks_dir = m
@@ -326,15 +524,16 @@ impl Config {
             .unwrap_or_else(|| daemon_dir.join("blocks"));
         let cookie = m.value_of("cookie").map(|s| s.to_owned());
 
-        let electrum_banner = m.value_of("electrum_banner").map_or_else(
-            || format!("Welcome to electrs-esplora {}", ELECTRS_VERSION),
-            |s| s.into(),
-        );
+        let electrum_banner = m
+            .value_of("electrum_banner")
+            .map_or_else(|| format!("Welcome to {}", *VERSION_STRING), |s| s.into());
 
         #[cfg(feature = "electrum-discovery")]
         let electrum_public_hosts = m
             .value_of("electrum_public_hosts")
             .map(|s| serde_json::from_str(s).expect("invalid --electrum-public-hosts"));
+        #[cfg(not(feature = "electrum-discovery"))]
+        let electrum_public_hosts: Option<crate::electrum::ServerHosts> = None;
 
         let mut log = stderrlog::new();
         log.verbosity(m.occurrences_of("verbosity") as usize);
@@ -347,6 +546,7 @@ impl Config {
         let config = Config {
             log,
             network_type,
+            magic,
             db_path,
             daemon_dir,
             blocks_dir,
@@ -358,19 +558,71 @@ impl Config {
             electrum_banner,
             http_addr,
             http_socket_file,
+            rpc_socket_file,
             monitoring_addr,
+            mempool_backlog_stats_ttl: value_t_or_exit!(m, "mempool_backlog_stats_ttl", u64),
+            mempool_recent_txs_size: value_t_or_exit!(m, "mempool_recent_txs_size", usize),
+            rest_default_block_limit: value_t_or_exit!(m, "rest_default_block_limit", usize),
+            rest_default_chain_txs_per_page: value_t_or_exit!(
+                m,
+                "rest_default_chain_txs_per_page",
+                usize
+            ),
+            rest_default_max_mempool_txs: value_t_or_exit!(
+                m,
+                "rest_default_max_mempool_txs",
+                usize
+            ),
+            rest_default_max_address_summary_txs: value_t_or_exit!(
+                m,
+                "rest_default_max_address_summary_txs",
+                usize
+            ),
+            rest_max_mempool_page_size: value_t_or_exit!(m, "rest_max_mempool_page_size", usize),
+            rest_max_mempool_txid_page_size: value_t_or_exit!(
+                m,
+                "rest_max_mempool_txid_page_size",
+                usize
+            ),
+            electrum_max_line_size: value_t_or_exit!(m, "electrum_max_line_size", usize),
+            electrum_max_subscriptions: value_t_or_exit!(m, "electrum_max_subscriptions", usize),
+            electrum_max_clients: value_t_or_exit!(m, "electrum_max_clients", usize),
+            electrum_idle_timeout: value_t_or_exit!(m, "electrum_idle_timeout", u64),
+            electrum_haproxy_depth: value_t_or_exit!(m, "electrum_haproxy_depth", usize),
+            electrum_connections_per_client: value_t_or_exit!(
+                m,
+                "electrum_connections_per_client",
+                usize
+            ),
             jsonrpc_import: m.is_present("jsonrpc_import"),
             light_mode: m.is_present("light_mode"),
+            main_loop_delay: value_t_or_exit!(m, "main_loop_delay", u64),
             address_search: m.is_present("address_search"),
+            index_unspendables: m.is_present("index_unspendables"),
             cors: m.value_of("cors").map(|s| s.to_string()),
             precache_scripts: m.value_of("precache_scripts").map(|s| s.to_string()),
+            precache_threads: m.value_of("precache_threads").map_or_else(
+                || {
+                    std::thread::available_parallelism()
+                        .expect("Can't get core count")
+                        .get()
+                        * 4
+                },
+                |s| match s.parse::<usize>() {
+                    Ok(v) if v > 0 => v,
+                    _ => clap::Error::value_validation_auto(format!(
+                        "The argument '{}' isn't a valid value",
+                        s
+                    ))
+                    .exit(),
+                },
+            ),
 
             #[cfg(feature = "liquid")]
             parent_network,
             #[cfg(feature = "liquid")]
             asset_db_path,
 
-            #[cfg(feature = "electrum-discovery")]
             electrum_public_hosts,
             #[cfg(feature = "electrum-discovery")]
             electrum_announce: m.is_present("electrum_announce"),

src/daemon.rs

@@ -2,15 +2,15 @@ use std::collections::{HashMap, HashSet};
 use std::io::{BufRead, BufReader, Lines, Write};
 use std::net::{SocketAddr, TcpStream};
 use std::path::PathBuf;
+use std::str::FromStr;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use base64;
-use bitcoin::hashes::hex::{FromHex, ToHex};
-use bitcoin::util::hash::BitcoinHash;
-use bitcoin::{BlockHash, Txid};
+use bitcoin::hashes::Hash;
 use glob;
 use hex;
+use itertools::Itertools;
 use serde_json::{from_str, from_value, Value};
 
 #[cfg(not(feature = "liquid"))]
@@ -18,7 +18,8 @@ use bitcoin::consensus::encode::{deserialize, serialize};
 #[cfg(feature = "liquid")]
 use elements::encode::{deserialize, serialize};
 
-use crate::chain::{Block, BlockHeader, Network, Transaction};
+use crate::chain::{Block, BlockHash, BlockHeader, Network, Transaction, Txid};
+use crate::config::BITCOIND_SUBVER;
 use crate::metrics::{HistogramOpts, HistogramVec, Metrics};
 use crate::signal::Waiter;
 use crate::util::HeaderList;
@@ -27,14 +28,14 @@ use crate::errors::*;
 
 fn parse_hash<T>(value: &Value) -> Result<T>
 where
-    T: FromHex,
+    T: FromStr,
+    <T as FromStr>::Err: std::fmt::Debug,
 {
-    Ok(T::from_hex(
-        value
-            .as_str()
-            .chain_err(|| format!("non-string value: {}", value))?,
-    )
-    .chain_err(|| format!("non-hex value: {}", value))?)
+    value
+        .as_str()
+        .chain_err(|| format!("non-string value: {}", value))?
+        .parse::<T>()
+        .map_err(|e| format!("failed to parse hash: {:?}", e).into())
 }
 
 fn header_from_value(value: Value) -> Result<BlockHeader> {
@@ -42,22 +43,19 @@ fn header_from_value(value: Value) -> Result<BlockHeader> {
         .as_str()
         .chain_err(|| format!("non-string header: {}", value))?;
     let header_bytes = hex::decode(header_hex).chain_err(|| "non-hex header")?;
-    Ok(
-        deserialize(&header_bytes)
-            .chain_err(|| format!("failed to parse header {}", header_hex))?,
-    )
+    deserialize(&header_bytes).chain_err(|| format!("failed to parse header {}", header_hex))
 }
 
 fn block_from_value(value: Value) -> Result<Block> {
     let block_hex = value.as_str().chain_err(|| "non-string block")?;
     let block_bytes = hex::decode(block_hex).chain_err(|| "non-hex block")?;
-    Ok(deserialize(&block_bytes).chain_err(|| format!("failed to parse block {}", block_hex))?)
+    deserialize(&block_bytes).chain_err(|| format!("failed to parse block {}", block_hex))
 }
 
 fn tx_from_value(value: Value) -> Result<Transaction> {
     let tx_hex = value.as_str().chain_err(|| "non-string tx")?;
     let tx_bytes = hex::decode(tx_hex).chain_err(|| "non-hex tx")?;
-    Ok(deserialize(&tx_bytes).chain_err(|| format!("failed to parse tx {}", tx_hex))?)
+    deserialize(&tx_bytes).chain_err(|| format!("failed to parse tx {}", tx_hex))
 }
 
 /// Parse JSONRPC error code, if exists.
@@ -69,7 +67,7 @@ fn parse_jsonrpc_reply(mut reply: Value, method: &str, expected_id: u64) -> Resu
     if let Some(reply_obj) = reply.as_object_mut() {
         if let Some(err) = reply_obj.get("error") {
             if !err.is_null() {
-                if let Some(code) = parse_error_code(&err) {
+                if let Some(code) = parse_error_code(err) {
                     match code {
                         // RPC_IN_WARMUP -> retry by later reconnection
                         -28 => bail!(ErrorKind::Connection(err.to_string())),
@@ -109,6 +107,11 @@ pub struct BlockchainInfo {
     pub initialblockdownload: Option<bool>,
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+pub struct MempoolInfo {
+    pub loaded: bool,
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 struct NetworkInfo {
     version: u64,
@@ -116,6 +119,54 @@ struct NetworkInfo {
     relayfee: f64, // in BTC/kB
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+struct MempoolFees {
+    base: f64,
+    #[serde(rename = "effective-feerate")]
+    effective_feerate: f64,
+    #[serde(rename = "effective-includes")]
+    effective_includes: Vec<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct MempoolAcceptResult {
+    txid: String,
+    wtxid: String,
+    allowed: Option<bool>,
+    vsize: Option<u32>,
+    fees: Option<MempoolFees>,
+    #[serde(rename = "reject-reason")]
+    reject_reason: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct MempoolFeesSubmitPackage {
+    base: f64,
+    #[serde(rename = "effective-feerate")]
+    effective_feerate: Option<f64>,
+    #[serde(rename = "effective-includes")]
+    effective_includes: Option<Vec<String>>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct SubmitPackageResult {
+    package_msg: String,
+    #[serde(rename = "tx-results")]
+    tx_results: HashMap<String, TxResult>,
+    #[serde(rename = "replaced-transactions")]
+    replaced_transactions: Option<Vec<String>>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct TxResult {
+    txid: String,
+    #[serde(rename = "other-wtxid")]
+    other_wtxid: Option<String>,
+    vsize: Option<u32>,
+    fees: Option<MempoolFeesSubmitPackage>,
+    error: Option<String>,
+}
+
 pub trait CookieGetter: Send + Sync {
     fn get(&self) -> Result<Vec<u8>>;
 }
@@ -188,7 +239,7 @@ impl Connection {
             .chain_err(|| {
                 ErrorKind::Connection("disconnected from daemon while receiving".to_owned())
             })?
-            .chain_err(|| "failed to read status")?;
+            .chain_err(|| ErrorKind::Connection("failed to read status".to_owned()))?;
         let mut headers = HashMap::new();
         for line in iter {
             let line = line.chain_err(|| ErrorKind::Connection("failed to read".to_owned()))?;
@@ -263,6 +314,7 @@ pub struct Daemon {
     daemon_dir: PathBuf,
     blocks_dir: PathBuf,
     network: Network,
+    magic: Option<u32>,
     conn: Mutex<Connection>,
     message_id: Counter, // for monotonic JSONRPC 'id'
     signal: Waiter,
@@ -273,19 +325,22 @@ pub struct Daemon {
 }
 
 impl Daemon {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
-        daemon_dir: &PathBuf,
-        blocks_dir: &PathBuf,
+        daemon_dir: PathBuf,
+        blocks_dir: PathBuf,
         daemon_rpc_addr: SocketAddr,
         cookie_getter: Arc<dyn CookieGetter>,
         network: Network,
+        magic: Option<u32>,
         signal: Waiter,
         metrics: &Metrics,
     ) -> Result<Daemon> {
         let daemon = Daemon {
-            daemon_dir: daemon_dir.clone(),
-            blocks_dir: blocks_dir.clone(),
+            daemon_dir,
+            blocks_dir,
             network,
+            magic,
             conn: Mutex::new(Connection::new(
                 daemon_rpc_addr,
                 cookie_getter,
@@ -310,6 +365,9 @@ impl Daemon {
                 network_info.subversion,
             )
         }
+        // Insert the subversion (/Satoshi xx.xx.xx(comment)/) string from bitcoind
+        _ = BITCOIND_SUBVER.set(network_info.subversion);
+
         let blockchain_info = daemon.getblockchaininfo()?;
         info!("{:?}", blockchain_info);
         if blockchain_info.pruned {
@@ -317,16 +375,24 @@ impl Daemon {
         }
         loop {
             let info = daemon.getblockchaininfo()?;
+            let mempool = daemon.getmempoolinfo()?;
 
-            if !info.initialblockdownload.unwrap_or(false) && info.blocks == info.headers {
+            let ibd_done = if network.is_regtest() {
+                info.blocks == info.headers
+            } else {
+                !info.initialblockdownload.unwrap_or(false)
+            };
+
+            if mempool.loaded && ibd_done && info.blocks == info.headers {
                 break;
             }
 
             warn!(
-                "waiting for bitcoind sync to finish: {}/{} blocks, verification progress: {:.3}%",
+                "waiting for bitcoind sync and mempool load to finish: {}/{} blocks, verification progress: {:.3}%, mempool loaded: {}",
                 info.blocks,
                 info.headers,
-                info.verificationprogress * 100.0
+                info.verificationprogress * 100.0,
+                mempool.loaded
             );
             signal.wait(Duration::from_secs(5), false)?;
         }
@@ -338,6 +404,7 @@ impl Daemon {
             daemon_dir: self.daemon_dir.clone(),
             blocks_dir: self.blocks_dir.clone(),
             network: self.network,
+            magic: self.magic,
             conn: Mutex::new(self.conn.lock().unwrap().reconnect()?),
             message_id: Counter::new(),
             signal: self.signal.clone(),
@@ -358,7 +425,7 @@ impl Daemon {
     }
 
     pub fn magic(&self) -> u32 {
-        self.network.magic()
+        self.magic.unwrap_or_else(|| self.network.magic())
     }
 
     fn call_jsonrpc(&self, method: &str, request: &Value) -> Result<Value> {
@@ -378,26 +445,63 @@ impl Daemon {
         Ok(result)
     }
 
-    fn handle_request_batch(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
+    fn handle_request_batch(
+        &self,
+        method: &str,
+        params_list: &[Value],
+        failure_threshold: f64,
+    ) -> Result<Vec<Value>> {
         let id = self.message_id.next();
-        let reqs = params_list
+        let chunks = params_list
             .iter()
             .map(|params| json!({"method": method, "params": params, "id": id}))
-            .collect();
+            .chunks(50_000); // Max Amount of batched requests
         let mut results = vec![];
-        let mut replies = self.call_jsonrpc(method, &reqs)?;
-        if let Some(replies_vec) = replies.as_array_mut() {
-            for reply in replies_vec {
-                results.push(parse_jsonrpc_reply(reply.take(), method, id)?)
+        let total_requests = params_list.len();
+        let mut failed_requests: u64 = 0;
+        let threshold = (failure_threshold * total_requests as f64).round() as u64;
+        let mut n = 0;
+
+        for chunk in &chunks {
+            let reqs = chunk.collect();
+            let mut replies = self.call_jsonrpc(method, &reqs)?;
+            if let Some(replies_vec) = replies.as_array_mut() {
+                for reply in replies_vec {
+                    n += 1;
+                    match parse_jsonrpc_reply(reply.take(), method, id) {
+                        Ok(parsed_reply) => results.push(parsed_reply),
+                        Err(e) => {
+                            failed_requests += 1;
+                            warn!(
+                                "batch request {} {}/{} failed: {}",
+                                method,
+                                n,
+                                total_requests,
+                                e.to_string()
+                            );
+                            // abort and return the last error once a threshold number of requests have failed
+                            if failed_requests > threshold {
+                                return Err(e);
+                            }
+                        }
+                    }
+                }
+            } else {
+                bail!("non-array replies: {:?}", replies);
             }
-            return Ok(results);
         }
-        bail!("non-array replies: {:?}", replies);
+
+        Ok(results)
     }
 
-    fn retry_request_batch(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
+    fn retry_request_batch(
+        &self,
+        method: &str,
+        params_list: &[Value],
+        failure_threshold: f64,
+    ) -> Result<Vec<Value>> {
         loop {
-            match self.handle_request_batch(method, params_list) {
+            match self.handle_request_batch(method, params_list, failure_threshold) {
                 Err(Error(ErrorKind::Connection(msg), _)) => {
                     warn!("reconnecting to bitcoind: {}", msg);
                     self.signal.wait(Duration::from_secs(3), false)?;
@@ -411,25 +515,30 @@ impl Daemon {
     }
 
     fn request(&self, method: &str, params: Value) -> Result<Value> {
-        let mut values = self.retry_request_batch(method, &[params])?;
+        let mut values = self.retry_request_batch(method, &[params], 0.0)?;
         assert_eq!(values.len(), 1);
         Ok(values.remove(0))
     }
 
     fn requests(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
-        self.retry_request_batch(method, params_list)
+        self.retry_request_batch(method, params_list, 0.0)
     }
 
     // bitcoind JSONRPC API:
 
     pub fn getblockchaininfo(&self) -> Result<BlockchainInfo> {
         let info: Value = self.request("getblockchaininfo", json!([]))?;
-        Ok(from_value(info).chain_err(|| "invalid blockchain info")?)
+        from_value(info).chain_err(|| "invalid blockchain info")
+    }
+
+    fn getmempoolinfo(&self) -> Result<MempoolInfo> {
+        let info: Value = self.request("getmempoolinfo", json!([]))?;
+        from_value(info).chain_err(|| "invalid mempool info")
     }
 
     fn getnetworkinfo(&self) -> Result<NetworkInfo> {
         let info: Value = self.request("getnetworkinfo", json!([]))?;
-        Ok(from_value(info).chain_err(|| "invalid network info")?)
+        from_value(info).chain_err(|| "invalid network info")
     }
 
     pub fn getbestblockhash(&self) -> Result<BlockHash> {
@@ -439,7 +548,7 @@ impl Daemon {
     pub fn getblockheader(&self, blockhash: &BlockHash) -> Result<BlockHeader> {
         header_from_value(self.request(
             "getblockheader",
-            json!([blockhash.to_hex(), /*verbose=*/ false]),
+            json!([blockhash.to_string(), /*verbose=*/ false]),
         )?)
     }
 
@@ -458,21 +567,22 @@ impl Daemon {
     }
 
     pub fn getblock(&self, blockhash: &BlockHash) -> Result<Block> {
-        let block = block_from_value(
-            self.request("getblock", json!([blockhash.to_hex(), /*verbose=*/ false]))?,
-        )?;
-        assert_eq!(block.bitcoin_hash(), *blockhash);
+        let block = block_from_value(self.request(
+            "getblock",
+            json!([blockhash.to_string(), /*verbose=*/ false]),
+        )?)?;
+        assert_eq!(block.block_hash(), *blockhash);
         Ok(block)
     }
 
     pub fn getblock_raw(&self, blockhash: &BlockHash, verbose: u32) -> Result<Value> {
-        self.request("getblock", json!([blockhash.to_hex(), verbose]))
+        self.request("getblock", json!([blockhash.to_string(), verbose]))
     }
 
     pub fn getblocks(&self, blockhashes: &[BlockHash]) -> Result<Vec<Block>> {
         let params_list: Vec<Value> = blockhashes
             .iter()
-            .map(|hash| json!([hash.to_hex(), /*verbose=*/ false]))
+            .map(|hash| json!([hash.to_string(), /*verbose=*/ false]))
             .collect();
         let values = self.requests("getblock", &params_list)?;
         let mut blocks = vec![];
@@ -485,15 +595,14 @@ impl Daemon {
     pub fn gettransactions(&self, txhashes: &[&Txid]) -> Result<Vec<Transaction>> {
         let params_list: Vec<Value> = txhashes
             .iter()
-            .map(|txhash| json!([txhash.to_hex(), /*verbose=*/ false]))
+            .map(|txhash| json!([txhash.to_string(), /*verbose=*/ false]))
             .collect();
-
-        let values = self.requests("getrawtransaction", &params_list)?;
+        let values = self.retry_request_batch("getrawtransaction", &params_list, 0.25)?;
         let mut txs = vec![];
         for value in values {
             txs.push(tx_from_value(value)?);
         }
-        assert_eq!(txhashes.len(), txs.len());
+        // missing transactions are skipped, so the number of txs returned may be less than the number of txids requested
         Ok(txs)
     }
 
@@ -505,21 +614,21 @@ impl Daemon {
     ) -> Result<Value> {
         self.request(
             "getrawtransaction",
-            json!([txid.to_hex(), verbose, blockhash]),
+            json!([txid.to_string(), verbose, blockhash]),
         )
     }
 
     pub fn getmempooltx(&self, txhash: &Txid) -> Result<Transaction> {
         let value = self.request(
             "getrawtransaction",
-            json!([txhash.to_hex(), /*verbose=*/ false]),
+            json!([txhash.to_string(), /*verbose=*/ false]),
         )?;
         tx_from_value(value)
     }
 
     pub fn getmempooltxids(&self) -> Result<HashSet<Txid>> {
         let res = self.request("getrawmempool", json!([/*verbose=*/ false]))?;
-        Ok(serde_json::from_value(res).chain_err(|| "invalid getrawmempool reply")?)
+        serde_json::from_value(res).chain_err(|| "invalid getrawmempool reply")
     }
 
     pub fn broadcast(&self, tx: &Transaction) -> Result<Txid> {
@@ -528,10 +637,43 @@ impl Daemon {
 
     pub fn broadcast_raw(&self, txhex: &str) -> Result<Txid> {
         let txid = self.request("sendrawtransaction", json!([txhex]))?;
-        Ok(
-            Txid::from_hex(txid.as_str().chain_err(|| "non-string txid")?)
-                .chain_err(|| "failed to parse txid")?,
-        )
+        txid.as_str()
+            .chain_err(|| "non-string txid")?
+            .parse::<Txid>()
+            .map_err(|e| format!("failed to parse txid: {:?}", e).into())
+    }
+
+    pub fn test_mempool_accept(
+        &self,
+        txhex: Vec<String>,
+        maxfeerate: Option<f64>,
+    ) -> Result<Vec<MempoolAcceptResult>> {
+        let params = match maxfeerate {
+            Some(rate) => json!([txhex, format!("{:.8}", rate)]),
+            None => json!([txhex]),
+        };
+        let result = self.request("testmempoolaccept", params)?;
+        serde_json::from_value::<Vec<MempoolAcceptResult>>(result)
+            .chain_err(|| "invalid testmempoolaccept reply")
+    }
+
+    pub fn submit_package(
+        &self,
+        txhex: Vec<String>,
+        maxfeerate: Option<f64>,
+        maxburnamount: Option<f64>,
+    ) -> Result<SubmitPackageResult> {
+        let params = match (maxfeerate, maxburnamount) {
+            (Some(rate), Some(burn)) => {
+                json!([txhex, format!("{:.8}", rate), format!("{:.8}", burn)])
+            }
+            (Some(rate), None) => json!([txhex, format!("{:.8}", rate)]),
+            (None, Some(burn)) => json!([txhex, null, format!("{:.8}", burn)]),
+            (None, None) => json!([txhex]),
+        };
+        let result = self.request("submitpackage", params)?;
+        serde_json::from_value::<SubmitPackageResult>(result)
+            .chain_err(|| "invalid submitpackage reply")
     }
 
     // Get estimated feerates for the provided confirmation targets using a batch RPC request
@@ -569,7 +711,7 @@ impl Daemon {
     }
 
     fn get_all_headers(&self, tip: &BlockHash) -> Result<Vec<BlockHeader>> {
-        let info: Value = self.request("getblockheader", json!([tip.to_hex()]))?;
+        let info: Value = self.request("getblockheader", json!([tip.to_string()]))?;
         let tip_height = info
             .get("height")
             .expect("missing height")
@@ -580,15 +722,15 @@ impl Daemon {
         let mut result = vec![];
         for heights in all_heights.chunks(chunk_size) {
             trace!("downloading {} block headers", heights.len());
-            let mut headers = self.getblockheaders(&heights)?;
+            let mut headers = self.getblockheaders(heights)?;
             assert!(headers.len() == heights.len());
             result.append(&mut headers);
         }
 
-        let mut blockhash = BlockHash::default();
+        let mut blockhash = BlockHash::all_zeros();
         for header in &result {
             assert_eq!(header.prev_blockhash, blockhash);
-            blockhash = header.bitcoin_hash();
+            blockhash = header.block_hash();
         }
         assert_eq!(blockhash, *tip);
         Ok(result)
@@ -611,7 +753,7 @@ impl Daemon {
             bestblockhash,
         );
         let mut new_headers = vec![];
-        let null_hash = BlockHash::default();
+        let null_hash = BlockHash::all_zeros();
         let mut blockhash = *bestblockhash;
         while blockhash != null_hash {
             if indexed_headers.header_by_blockhash(&blockhash).is_some() {

src/electrum/client.rs

@@ -1,66 +1,15 @@
 use std::collections::HashMap;
-use std::convert::{TryFrom, TryInto};
-use std::net::ToSocketAddrs;
+use std::convert::TryFrom;
 
-use bitcoin::{hashes::Hash, BlockHash};
-use electrum_client::client::{
-    Client as RClient, ElectrumPlaintextStream, ElectrumProxyStream, ElectrumSslStream,
-};
-pub use electrum_client::types::ServerFeaturesRes;
-pub use electrum_client::Error as ElectrumError;
+use bitcoin::hashes::sha256d;
+use bitcoin::hashes::Hash;
+pub use electrum_client::client::Client;
+pub use electrum_client::ServerFeaturesRes;
 
+use crate::chain::BlockHash;
 use crate::electrum::ServerFeatures;
 use crate::errors::{Error, ResultExt};
 
-pub enum Client {
-    Tcp(RClient<ElectrumPlaintextStream>),
-    Ssl(RClient<ElectrumSslStream>),
-    ProxyTcp(RClient<ElectrumProxyStream>),
-    // proxy+ssl on the same connection appears to be unsupported by the electrum_client crate
-}
-
-// impl<S: Read + Write> Client<S> {
-
-impl Client {
-    pub fn new<A: ToSocketAddrs>(socket_addr: A) -> Result<Self, ElectrumError> {
-        Ok(Client::Tcp(RClient::new(socket_addr)?))
-    }
-
-    pub fn new_ssl(domain_addr: (&str, u16)) -> Result<Self, ElectrumError> {
-        // SSL certificates are not validated
-        Ok(Client::Ssl(RClient::new_ssl(domain_addr, false)?))
-        // XXX this should ideally use the previously resolved IP address instead of the hostname, which
-        // shuold be possible because we don't validate, but it appears like rustls does not support this.
-    }
-
-    pub fn new_proxy<A: ToSocketAddrs>(
-        target_addr: (&str, u16),
-        proxy_addr: A,
-    ) -> Result<Self, ElectrumError> {
-        Ok(Client::ProxyTcp(RClient::new_proxy(
-            target_addr,
-            proxy_addr,
-        )?))
-    }
-
-    pub fn server_features(&mut self) -> Result<ServerFeatures, Error> {
-        match self {
-            Client::Tcp(c) => c.server_features(),
-            Client::Ssl(c) => c.server_features(),
-            Client::ProxyTcp(c) => c.server_features(),
-        }?
-        .try_into()
-    }
-
-    pub fn server_add_peer(&mut self, features: &ServerFeatures) -> Result<bool, ElectrumError> {
-        match self {
-            Client::Tcp(c) => c.server_add_peer(features),
-            Client::Ssl(c) => c.server_add_peer(features),
-            Client::ProxyTcp(c) => c.server_add_peer(features),
-        }
-    }
-}
-
 // Convert from electrum-client's server features struct to ours. We're using a different struct because
 // the electrum-client's one doesn't support the "hosts" key.
 impl TryFrom<ServerFeaturesRes> for ServerFeatures {
@@ -71,7 +20,9 @@ impl TryFrom<ServerFeaturesRes> for ServerFeatures {
         Ok(ServerFeatures {
             // electrum-client doesn't retain the hosts map data, but we already have it from the add_peer request
             hosts: HashMap::new(),
-            genesis_hash: BlockHash::from_inner(features.genesis_hash),
+            genesis_hash: BlockHash::from_raw_hash(sha256d::Hash::from_byte_array(
+                features.genesis_hash,
+            )),
             server_version: features.server_version,
             protocol_min: features
                 .protocol_min

src/electrum/discovery.rs

@@ -1,5 +1,6 @@
 use std::cmp::Ordering;
 use std::collections::{hash_map::Entry, BinaryHeap, HashMap, HashSet};
+use std::convert::TryInto;
 use std::fmt;
 use std::net::{IpAddr, SocketAddr, ToSocketAddrs};
 use std::str::FromStr;
@@ -7,7 +8,7 @@ use std::sync::{Arc, RwLock};
 use std::thread;
 use std::time::{Duration, Instant};
 
-use bitcoin::BlockHash;
+use electrum_client::ElectrumApi;
 
 use crate::chain::Network;
 use crate::electrum::{Client, Hostname, Port, ProtocolVersion, ServerFeatures};
@@ -32,9 +33,6 @@ pub struct DiscoveryManager {
     /// A list of servers that were found to be healthy on their last health check
     healthy: RwLock<HashMap<ServerAddr, Server>>,
 
-    /// Used to test for network compatibility
-    our_genesis_hash: BlockHash,
-
     /// Used to test for protocol version compatibility
     our_version: ProtocolVersion,
 
@@ -82,6 +80,7 @@ struct HealthCheck {
     hostname: Hostname,
     service: Service,
     is_default: bool,
+    #[allow(dead_code)]
     added_by: Option<IpAddr>,
     last_check: Option<Instant>,
     last_healthy: Option<Instant>,
@@ -110,7 +109,6 @@ impl DiscoveryManager {
             })
             .collect();
         let discovery = Self {
-            our_genesis_hash: our_network.genesis_hash(),
             our_addrs,
             our_version,
             our_features,
@@ -185,7 +183,7 @@ impl DiscoveryManager {
                     .filter(|service| {
                         existing_services
                             .get(&addr)
-                            .map_or(true, |s| !s.contains(service))
+                            .is_none_or(|s| !s.contains(service))
                     })
                     .map(|service| {
                         HealthCheck::new(addr.clone(), hostname.clone(), service, Some(added_by))
@@ -237,9 +235,9 @@ impl DiscoveryManager {
     /// Run the next health check in the queue (a single one)
     fn run_health_check(&self) -> Result<()> {
         // abort if there are no entries in the queue, or its still too early for the next one up
-        if self.queue.read().unwrap().peek().map_or(true, |next| {
+        if self.queue.read().unwrap().peek().is_none_or(|next| {
             next.last_check
-                .map_or(false, |t| t.elapsed() < HEALTH_CHECK_FREQ)
+                .is_some_and(|t| t.elapsed() < HEALTH_CHECK_FREQ)
         }) {
             return Ok(());
         }
@@ -322,19 +320,29 @@ impl DiscoveryManager {
     ) -> Result<ServerFeatures> {
         debug!("checking service {:?} {:?}", addr, service);
 
-        let mut client: Client = match (addr, service) {
-            (ServerAddr::Clearnet(ip), Service::Tcp(port)) => Client::new((*ip, port))?,
-            (ServerAddr::Clearnet(_), Service::Ssl(port)) => Client::new_ssl((hostname, port))?,
-            (ServerAddr::Onion(hostname), Service::Tcp(port)) => {
-                let tor_proxy = self
-                    .tor_proxy
-                    .chain_err(|| "no tor proxy configured, onion hosts are unsupported")?;
-                Client::new_proxy((hostname, port), tor_proxy)?
+        let server_url = match (addr, service) {
+            (ServerAddr::Clearnet(ip), Service::Tcp(port)) => format!("tcp://{}:{}", ip, port),
+            (ServerAddr::Clearnet(_), Service::Ssl(port)) => format!("ssl://{}:{}", hostname, port),
+            (ServerAddr::Onion(onion_host), Service::Tcp(port)) => {
+                format!("tcp://{}:{}", onion_host, port)
+            }
+            (ServerAddr::Onion(onion_host), Service::Ssl(port)) => {
+                format!("ssl://{}:{}", onion_host, port)
             }
-            (ServerAddr::Onion(_), Service::Ssl(_)) => bail!("ssl over onion is unsupported"),
         };
 
-        let features = client.server_features()?;
+        let mut config = electrum_client::ConfigBuilder::new();
+        if let ServerAddr::Onion(_) = addr {
+            let socks = electrum_client::Socks5Config::new(
+                self.tor_proxy
+                    .chain_err(|| "no tor proxy configured, onion hosts are unsupported")?,
+            );
+            config = config.socks5(Some(socks))
+        }
+
+        let client = Client::from_config(&server_url, config.build())?;
+
+        let features = client.server_features()?.try_into()?;
         self.verify_compatibility(&features)?;
 
         if self.announce {
@@ -350,7 +358,7 @@ impl DiscoveryManager {
 
     fn verify_compatibility(&self, features: &ServerFeatures) -> Result<()> {
         ensure!(
-            features.genesis_hash == self.our_genesis_hash,
+            features.genesis_hash == self.our_features.genesis_hash,
             "incompatible networks"
         );
 
@@ -514,36 +522,67 @@ fn is_remote_addr(addr: &ServerAddr) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chain::genesis_hash;
     use crate::chain::Network;
     use std::time;
 
+    use crate::config::VERSION_STRING;
+
+    const PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::new(1, 4);
+
     #[test]
+    #[ignore = "This test requires external connection to server that no longer exists"]
     fn test() -> Result<()> {
         stderrlog::new().verbosity(4).init().unwrap();
 
-        let discovery = DiscoveryManager::new(
-            Network::Testnet,
-            "1.4".parse().unwrap(),
-            Some("127.0.0.1:9150".parse().unwrap()),
-        );
+        #[cfg(feature = "liquid")]
+        let testnet = Network::LiquidTestnet;
+        #[cfg(not(feature = "liquid"))]
+        let testnet = Network::Testnet;
 
-        discovery.add_default_server(
-            "electrum.blockstream.info".into(),
-            vec![Service::Tcp(60001)],
-        );
-        discovery.add_default_server("testnet.hsmiths.com".into(), vec![Service::Ssl(53012)]);
-        discovery.add_default_server(
-            "tn.not.fyi".into(),
-            vec![Service::Tcp(55001), Service::Ssl(55002)],
-        );
-        discovery.add_default_server(
-            "electrum.blockstream.info".into(),
-            vec![Service::Tcp(60001), Service::Ssl(60002)],
-        );
-        discovery.add_default_server(
-            "explorerzydxu5ecjrkwceayqybizmpjjznk5izmitf2modhcusuqlid.onion".into(),
-            vec![Service::Tcp(143)],
-        );
+        let features = ServerFeatures {
+            hosts: serde_json::from_str("{\"test.foobar.example\":{\"tcp_port\":60002}}").unwrap(),
+            server_version: VERSION_STRING.clone(),
+            genesis_hash: genesis_hash(testnet),
+            protocol_min: PROTOCOL_VERSION,
+            protocol_max: PROTOCOL_VERSION,
+            hash_function: "sha256".into(),
+            pruning: None,
+        };
+        let discovery = Arc::new(DiscoveryManager::new(
+            testnet,
+            features,
+            PROTOCOL_VERSION,
+            false,
+            None,
+        ));
+        discovery
+            .add_default_server(
+                "electrum.blockstream.info".into(),
+                vec![Service::Tcp(60001)],
+            )
+            .unwrap();
+        discovery
+            .add_default_server("testnet.hsmiths.com".into(), vec![Service::Ssl(53012)])
+            .unwrap();
+        discovery
+            .add_default_server(
+                "tn.not.fyi".into(),
+                vec![Service::Tcp(55001), Service::Ssl(55002)],
+            )
+            .unwrap();
+        discovery
+            .add_default_server(
+                "electrum.blockstream.info".into(),
+                vec![Service::Tcp(60001), Service::Ssl(60002)],
+            )
+            .unwrap();
+        discovery
+            .add_default_server(
+                "explorerzydxu5ecjrkwceayqybizmpjjznk5izmitf2modhcusuqlid.onion".into(),
+                vec![Service::Tcp(143)],
+            )
+            .unwrap();
 
         debug!("{:#?}", discovery);
 

src/electrum/discovery/default_servers.rs

@@ -1,8 +1,11 @@
 use crate::chain::Network;
+#[allow(unused_imports)]
 use crate::electrum::discovery::{DiscoveryManager, Service};
 
+#[allow(unused_variables)]
 pub fn add_default_servers(discovery: &DiscoveryManager, network: Network) {
     match network {
+        #[cfg(not(feature = "liquid"))]
         Network::Bitcoin => {
             discovery
                 .add_default_server(
@@ -401,6 +404,7 @@ pub fn add_default_servers(discovery: &DiscoveryManager, network: Network) {
                 )
                 .ok();
         }
+        #[cfg(not(feature = "liquid"))]
         Network::Testnet => {
             discovery
                 .add_default_server(

src/electrum/mod.rs

@@ -14,8 +14,7 @@ use std::str::FromStr;
 
 use serde::{de, Deserialize, Deserializer, Serialize};
 
-use bitcoin::BlockHash;
-
+use crate::chain::BlockHash;
 use crate::errors::ResultExt;
 use crate::util::BlockId;
 

src/elements/asset.rs

@@ -1,25 +1,34 @@
 use std::collections::{HashMap, HashSet};
+use std::sync::{Arc, RwLock, RwLockReadGuard};
 
-use bitcoin::hashes::{hex::FromHex, sha256, Hash};
-use bitcoin::{BlockHash, Txid};
+use bitcoin::hashes::{sha256, Hash};
 use elements::confidential::{Asset, Value};
 use elements::encode::{deserialize, serialize};
+use elements::secp256k1_zkp::ZERO_TWEAK;
 use elements::{issuance::ContractHash, AssetId, AssetIssuance, OutPoint, Transaction, TxIn};
 
-use crate::chain::Network;
+use crate::chain::{BNetwork, BlockHash, Network, Txid};
 use crate::elements::peg::{get_pegin_data, get_pegout_data, PeginInfo, PegoutInfo};
 use crate::elements::registry::{AssetMeta, AssetRegistry};
 use crate::errors::*;
-use crate::new_index::schema::{TxHistoryInfo, TxHistoryKey, TxHistoryRow};
+use crate::new_index::schema::{Operation, TxHistoryInfo, TxHistoryKey, TxHistoryRow};
 use crate::new_index::{db::DBFlush, ChainQuery, DBRow, Mempool, Query};
-use crate::util::{full_hash, Bytes, FullHash, TransactionStatus, TxInput};
+use crate::util::{
+    bincode_util, full_hash, Bytes, FullHash, IsProvablyUnspendable, TransactionStatus, TxInput,
+};
 
 lazy_static! {
     pub static ref NATIVE_ASSET_ID: AssetId =
-        AssetId::from_hex("6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d")
+        "6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d"
+            .parse()
             .unwrap();
     pub static ref NATIVE_ASSET_ID_TESTNET: AssetId =
-        AssetId::from_hex("5ac9f65c0efcc4775e0baec4ec03abdde22473cd3cf33c0419ca290e0751b225")
+        "144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49"
+            .parse()
+            .unwrap();
+    pub static ref NATIVE_ASSET_ID_REGTEST: AssetId =
+        "5ac9f65c0efcc4775e0baec4ec03abdde22473cd3cf33c0419ca290e0751b225"
+            .parse()
             .unwrap();
 }
 
@@ -29,22 +38,24 @@ fn parse_asset_id(sl: &[u8]) -> AssetId {
 
 #[derive(Serialize)]
 #[serde(untagged)]
+#[allow(clippy::large_enum_variant)]
 pub enum LiquidAsset {
     Issued(IssuedAsset),
-    Native(NativeAsset),
+    Native(PeggedAsset),
 }
 
 #[derive(Serialize)]
-pub struct NativeAsset {
+pub struct PeggedAsset {
     pub asset_id: AssetId,
-    pub chain_stats: NativeAssetStats,
-    pub mempool_stats: NativeAssetStats,
+    pub chain_stats: PeggedAssetStats,
+    pub mempool_stats: PeggedAssetStats,
 }
 
 #[derive(Serialize)]
 pub struct IssuedAsset {
     pub asset_id: AssetId,
     pub issuance_txin: TxInput,
+    #[serde(serialize_with = "crate::util::serialize_outpoint")]
     pub issuance_prevout: OutPoint,
     pub reissuance_token: AssetId,
 
@@ -66,9 +77,9 @@ pub struct IssuedAsset {
 #[derive(Serialize, Deserialize, Debug)]
 pub struct AssetRow {
     pub issuance_txid: FullHash,
-    pub issuance_vin: u16,
+    pub issuance_vin: u32,
     pub prev_txid: FullHash,
-    pub prev_vout: u16,
+    pub prev_vout: u32,
     pub issuance: Bytes, // bincode does not like dealing with AssetIssuance, deserialization fails with "invalid type: sequence, expected a struct"
     pub reissuance_token: FullHash,
 }
@@ -87,7 +98,7 @@ impl IssuedAsset {
         let reissuance_token = parse_asset_id(&asset.reissuance_token);
 
         let contract_hash = if issuance.asset_entropy != [0u8; 32] {
-            Some(ContractHash::from_inner(issuance.asset_entropy))
+            Some(ContractHash::from_byte_array(issuance.asset_entropy))
         } else {
             None
         };
@@ -100,7 +111,7 @@ impl IssuedAsset {
             },
             issuance_prevout: OutPoint {
                 txid: deserialize(&asset.prev_txid).unwrap(),
-                vout: asset.prev_vout as u32,
+                vout: asset.prev_vout,
             },
             contract_hash,
             reissuance_token,
@@ -147,9 +158,10 @@ impl LiquidAsset {
 }
 
 #[derive(Serialize, Deserialize, Debug)]
+#[cfg_attr(test, derive(PartialEq, Eq))]
 pub struct IssuingInfo {
     pub txid: FullHash,
-    pub vin: u16,
+    pub vin: u32,
     pub is_reissuance: bool,
     // None for blinded issuances
     pub issued_amount: Option<u64>,
@@ -157,9 +169,10 @@ pub struct IssuingInfo {
 }
 
 #[derive(Serialize, Deserialize, Debug)]
+#[cfg_attr(test, derive(PartialEq, Eq))]
 pub struct BurningInfo {
     pub txid: FullHash,
-    pub vout: u16,
+    pub vout: u32,
     pub value: u64,
 }
 
@@ -167,24 +180,29 @@ pub struct BurningInfo {
 pub fn index_confirmed_tx_assets(
     tx: &Transaction,
     confirmed_height: u32,
+    tx_position: u16,
     network: Network,
-    parent_network: Network,
+    parent_network: BNetwork,
     rows: &mut Vec<DBRow>,
+    op: &Operation,
 ) {
     let (history, issuances) = index_tx_assets(tx, network, parent_network);
 
-    rows.extend(
-        history.into_iter().map(|(asset_id, info)| {
-            asset_history_row(&asset_id, confirmed_height, info).into_row()
-        }),
-    );
+    rows.extend(history.into_iter().map(|(asset_id, info)| {
+        let history_row = asset_history_row(&asset_id, confirmed_height, tx_position, info);
+        if let Operation::DeleteBlocksWithHistory(tx) = op {
+            tx.send(history_row.key.hash)
+                .expect("unbounded channel won't fail");
+        }
+        history_row.into_row()
+    }));
 
     // the initial issuance is kept twice: once in the history index under I<asset><height><txid:vin>,
     // and once separately under i<asset> for asset lookup with some more associated metadata.
     // reissuances are only kept under the history index.
     rows.extend(issuances.into_iter().map(|(asset_id, asset_row)| DBRow {
         key: [b"i", &asset_id.into_inner()[..]].concat(),
-        value: bincode::serialize(&asset_row).unwrap(),
+        value: bincode_util::serialize_little(&asset_row).unwrap(),
     }));
 }
 
@@ -192,16 +210,13 @@ pub fn index_confirmed_tx_assets(
 pub fn index_mempool_tx_assets(
     tx: &Transaction,
     network: Network,
-    parent_network: Network,
+    parent_network: BNetwork,
     asset_history: &mut HashMap<AssetId, Vec<TxHistoryInfo>>,
     asset_issuance: &mut HashMap<AssetId, AssetRow>,
 ) {
     let (history, issuances) = index_tx_assets(tx, network, parent_network);
     for (asset_id, info) in history {
-        asset_history
-            .entry(asset_id)
-            .or_insert_with(Vec::new)
-            .push(info);
+        asset_history.entry(asset_id).or_default().push(info);
     }
     for (asset_id, issuance) in issuances {
         asset_issuance.insert(asset_id, issuance);
@@ -226,12 +241,13 @@ pub fn remove_mempool_tx_assets(
     });
 }
 
+type HistoryAndIssuances = (Vec<(AssetId, TxHistoryInfo)>, Vec<(AssetId, AssetRow)>);
 // Internal utility function, index a transaction and return its history entries and issuances
 fn index_tx_assets(
     tx: &Transaction,
     network: Network,
-    parent_network: Network,
-) -> (Vec<(AssetId, TxHistoryInfo)>, Vec<(AssetId, AssetRow)>) {
+    parent_network: BNetwork,
+) -> HistoryAndIssuances {
     let mut history = vec![];
     let mut issuances = vec![];
 
@@ -243,19 +259,19 @@ fn index_tx_assets(
                 pegout.asset.explicit().unwrap(),
                 TxHistoryInfo::Pegout(PegoutInfo {
                     txid,
-                    vout: txo_index as u16,
+                    vout: txo_index as u32,
                     value: pegout.value,
                 }),
             ));
-        } else if txo.script_pubkey.is_provably_unspendable() {
+        } else if txo.script_pubkey.is_provably_unspendable_() && !txo.is_fee() {
             if let (Asset::Explicit(asset_id), Value::Explicit(value)) = (txo.asset, txo.value) {
                 if value > 0 {
                     history.push((
                         asset_id,
                         TxHistoryInfo::Burning(BurningInfo {
                             txid,
-                            vout: txo_index as u16,
-                            value: value,
+                            vout: txo_index as u32,
+                            value,
                         }),
                     ));
                 }
@@ -266,15 +282,15 @@ fn index_tx_assets(
     for (txi_index, txi) in tx.input.iter().enumerate() {
         if let Some(pegin) = get_pegin_data(txi, network) {
             history.push((
-                pegin.asset.explicit().unwrap(),
+                pegin.asset,
                 TxHistoryInfo::Pegin(PeginInfo {
                     txid,
-                    vin: txi_index as u16,
+                    vin: txi_index as u32,
                     value: pegin.value,
                 }),
             ));
         } else if txi.has_issuance() {
-            let is_reissuance = txi.asset_issuance.asset_blinding_nonce != [0u8; 32];
+            let is_reissuance = txi.asset_issuance.asset_blinding_nonce != ZERO_TWEAK;
 
             let asset_entropy = get_issuance_entropy(txi).expect("invalid issuance");
             let asset_id = AssetId::from_entropy(asset_entropy);
@@ -294,7 +310,7 @@ fn index_tx_assets(
                 asset_id,
                 TxHistoryInfo::Issuing(IssuingInfo {
                     txid,
-                    vin: txi_index as u16,
+                    vin: txi_index as u32,
                     is_reissuance,
                     issued_amount,
                     token_amount,
@@ -302,10 +318,8 @@ fn index_tx_assets(
             ));
 
             if !is_reissuance {
-                let is_confidential = match txi.asset_issuance.inflation_keys {
-                    Value::Confidential(..) => true,
-                    _ => false,
-                };
+                let is_confidential =
+                    matches!(txi.asset_issuance.inflation_keys, Value::Confidential(..));
                 let reissuance_token =
                     AssetId::reissuance_token_from_entropy(asset_entropy, is_confidential);
 
@@ -313,9 +327,9 @@ fn index_tx_assets(
                     asset_id,
                     AssetRow {
                         issuance_txid: txid,
-                        issuance_vin: txi_index as u16,
+                        issuance_vin: txi_index as u32,
                         prev_txid: full_hash(&txi.previous_output.txid[..]),
-                        prev_vout: txi.previous_output.vout as u16,
+                        prev_vout: txi.previous_output.vout,
                         issuance: serialize(&txi.asset_issuance),
                         reissuance_token: full_hash(&reissuance_token.into_inner()[..]),
                     },
@@ -330,38 +344,49 @@ fn index_tx_assets(
 fn asset_history_row(
     asset_id: &AssetId,
     confirmed_height: u32,
+    tx_position: u16,
     txinfo: TxHistoryInfo,
 ) -> TxHistoryRow {
     let key = TxHistoryKey {
         code: b'I',
         hash: full_hash(&asset_id.into_inner()[..]),
         confirmed_height,
+        tx_position,
         txinfo,
     };
     TxHistoryRow { key }
 }
 
+pub enum AssetRegistryLock<'a> {
+    RwLock(&'a Arc<RwLock<AssetRegistry>>),
+    RwLockReadGuard(&'a RwLockReadGuard<'a, AssetRegistry>),
+}
+
 pub fn lookup_asset(
     query: &Query,
-    registry: Option<&AssetRegistry>,
+    registry: Option<AssetRegistryLock>,
     asset_id: &AssetId,
+    meta: Option<&AssetMeta>, // may optionally be provided if already known
 ) -> Result<Option<LiquidAsset>> {
-    if asset_id == query.network().native_asset() {
-        let (chain_stats, mempool_stats) = native_asset_stats(query);
+    if query.network().pegged_asset() == Some(asset_id) {
+        let (chain_stats, mempool_stats) = pegged_asset_stats(query, asset_id);
 
-        return Ok(Some(LiquidAsset::Native(NativeAsset {
+        return Ok(Some(LiquidAsset::Native(PeggedAsset {
             asset_id: *asset_id,
-            chain_stats: chain_stats,
-            mempool_stats: mempool_stats,
+            chain_stats,
+            mempool_stats,
         })));
     }
 
     let history_db = query.chain().store().history_db();
-    let mempool_issuances = &query.mempool().asset_issuance;
+    let mempool = query.mempool();
+    let mempool_issuances = &mempool.asset_issuance;
 
     let chain_row = history_db
         .get(&[b"i", &asset_id.into_inner()[..]].concat())
-        .map(|row| bincode::deserialize::<AssetRow>(&row).expect("failed parsing AssetRow"));
+        .map(|row| {
+            bincode_util::deserialize_little::<AssetRow>(&row).expect("failed parsing AssetRow")
+        });
 
     let row = chain_row
         .as_ref()
@@ -370,8 +395,14 @@ pub fn lookup_asset(
     Ok(if let Some(row) = row {
         let reissuance_token = parse_asset_id(&row.reissuance_token);
 
-        let meta = registry.map_or_else(|| Ok(None), |r| r.load(asset_id))?;
-        let stats = issued_asset_stats(query, asset_id, &reissuance_token);
+        let meta = meta.cloned().or_else(|| match registry {
+            Some(AssetRegistryLock::RwLock(rwlock)) => {
+                rwlock.read().unwrap().get(asset_id).cloned()
+            }
+            Some(AssetRegistryLock::RwLockReadGuard(guard)) => guard.get(asset_id).cloned(),
+            None => None,
+        });
+        let stats = issued_asset_stats(query.chain(), &mempool, asset_id, &reissuance_token);
         let status = query.get_tx_status(&deserialize(&row.issuance_txid).unwrap());
 
         let asset = IssuedAsset::new(asset_id, row, stats, meta, status);
@@ -383,11 +414,11 @@ pub fn lookup_asset(
 }
 
 pub fn get_issuance_entropy(txin: &TxIn) -> Result<sha256::Midstate> {
-    if !txin.has_issuance {
+    if !txin.has_issuance() {
         bail!("input has no issuance");
     }
 
-    let is_reissuance = txin.asset_issuance.asset_blinding_nonce != [0u8; 32];
+    let is_reissuance = txin.asset_issuance.asset_blinding_nonce != ZERO_TWEAK;
 
     Ok(if !is_reissuance {
         let contract_hash = ContractHash::from_slice(&txin.asset_issuance.asset_entropy)
@@ -403,7 +434,7 @@ pub fn get_issuance_entropy(txin: &TxIn) -> Result<sha256::Midstate> {
 // Asset stats
 //
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Default)]
 pub struct IssuedAssetStats {
     pub tx_count: usize,
     pub issuance_count: usize,
@@ -414,22 +445,8 @@ pub struct IssuedAssetStats {
     pub burned_reissuance_tokens: u64,
 }
 
-impl Default for IssuedAssetStats {
-    fn default() -> Self {
-        Self {
-            tx_count: 0,
-            issuance_count: 0,
-            issued_amount: 0,
-            burned_amount: 0,
-            has_blinded_issuances: false,
-            reissuance_tokens: None,
-            burned_reissuance_tokens: 0,
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct NativeAssetStats {
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct PeggedAssetStats {
     pub tx_count: usize,
     pub peg_in_count: usize,
     pub peg_in_amount: u64,
@@ -439,20 +456,6 @@ pub struct NativeAssetStats {
     pub burned_amount: u64,
 }
 
-impl Default for NativeAssetStats {
-    fn default() -> Self {
-        Self {
-            tx_count: 0,
-            peg_in_count: 0,
-            peg_in_amount: 0,
-            peg_out_count: 0,
-            peg_out_amount: 0,
-            burn_count: 0,
-            burned_amount: 0,
-        }
-    }
-}
-
 type AssetStatApplyFn<T> = fn(&TxHistoryInfo, &mut T, &mut HashSet<Txid>);
 
 fn asset_cache_key(asset_id: &AssetId) -> Bytes {
@@ -465,42 +468,39 @@ where
 {
     DBRow {
         key: asset_cache_key(asset_id),
-        value: bincode::serialize(&(stats, blockhash)).unwrap(),
+        value: bincode_util::serialize_little(&(stats, blockhash)).unwrap(),
     }
 }
 
-// Get stats for the network's native asset
-fn native_asset_stats(query: &Query) -> (NativeAssetStats, NativeAssetStats) {
-    let asset_id = query.network().native_asset();
-
+// Get stats for the network's pegged asset
+fn pegged_asset_stats(query: &Query, asset_id: &AssetId) -> (PeggedAssetStats, PeggedAssetStats) {
     (
-        chain_asset_stats(query.chain(), asset_id, apply_native_asset_stats),
-        mempool_asset_stats(&query.mempool(), asset_id, apply_native_asset_stats),
+        chain_asset_stats(query.chain(), asset_id, apply_pegged_asset_stats),
+        mempool_asset_stats(&query.mempool(), asset_id, apply_pegged_asset_stats),
     )
 }
 
-// Get stats for user-issued assets
+// Get stats for issued assets
 fn issued_asset_stats(
-    query: &Query,
+    chain: &ChainQuery,
+    mempool: &Mempool,
     asset_id: &AssetId,
     reissuance_token: &AssetId,
 ) -> (IssuedAssetStats, IssuedAssetStats) {
     let afn = apply_issued_asset_stats;
 
-    let chain = query.chain();
     let mut chain_stats = chain_asset_stats(chain, asset_id, afn);
     chain_stats.burned_reissuance_tokens =
         chain_asset_stats(chain, reissuance_token, afn).burned_amount;
 
-    let mempool = query.mempool();
-    let mut mempool_stats = mempool_asset_stats(&mempool, &asset_id, afn);
+    let mut mempool_stats = mempool_asset_stats(mempool, asset_id, afn);
     mempool_stats.burned_reissuance_tokens =
-        mempool_asset_stats(&mempool, &reissuance_token, afn).burned_amount;
+        mempool_asset_stats(mempool, reissuance_token, afn).burned_amount;
 
     (chain_stats, mempool_stats)
 }
 
-// Get on-chain confirmed asset stats (user-issued or the native asset)
+// Get on-chain confirmed asset stats (issued or the pegged asset)
 fn chain_asset_stats<T>(chain: &ChainQuery, asset_id: &AssetId, apply_fn: AssetStatApplyFn<T>) -> T
 where
     T: Default + serde::Serialize + serde::de::DeserializeOwned,
@@ -511,7 +511,7 @@ where
         .store()
         .cache_db()
         .get(&asset_cache_key(asset_id))
-        .map(|c| bincode::deserialize(&c).unwrap())
+        .map(|c| bincode_util::deserialize_little(&c).unwrap())
         .and_then(|(stats, blockhash)| {
             chain
                 .height_by_hash(&blockhash)
@@ -569,7 +569,7 @@ fn chain_asset_stats_delta<T>(
     (stats, lastblock)
 }
 
-// Get mempool asset stats (user-issued or the native asset)
+// Get mempool asset stats (issued or the pegged asset)
 pub fn mempool_asset_stats<T>(
     mempool: &Mempool,
     asset_id: &AssetId,
@@ -629,9 +629,9 @@ fn apply_issued_asset_stats(
     }
 }
 
-fn apply_native_asset_stats(
+fn apply_pegged_asset_stats(
     info: &TxHistoryInfo,
-    stats: &mut NativeAssetStats,
+    stats: &mut PeggedAssetStats,
     seen_txids: &mut HashSet<Txid>,
 ) {
     if seen_txids.insert(info.get_txid()) {

src/elements/mod.rs

@@ -1,4 +1,6 @@
-use bitcoin::hashes::{hex::ToHex, Hash};
+use bitcoin::hashes::Hash;
+use elements::hex::ToHex;
+use elements::secp256k1_zkp::ZERO_TWEAK;
 use elements::{confidential::Value, encode::serialize, issuance::ContractHash, AssetId, TxIn};
 
 pub mod asset;
@@ -7,7 +9,7 @@ mod registry;
 
 use asset::get_issuance_entropy;
 pub use asset::{lookup_asset, LiquidAsset};
-pub use registry::AssetRegistry;
+pub use registry::{AssetMeta, AssetRegistry, AssetSorting};
 
 #[derive(Serialize, Deserialize, Clone)]
 pub struct IssuanceValue {
@@ -31,7 +33,7 @@ pub struct IssuanceValue {
 impl From<&TxIn> for IssuanceValue {
     fn from(txin: &TxIn) -> Self {
         let issuance = &txin.asset_issuance;
-        let is_reissuance = issuance.asset_blinding_nonce != [0u8; 32];
+        let is_reissuance = issuance.asset_blinding_nonce != ZERO_TWEAK;
 
         let asset_entropy = get_issuance_entropy(txin).expect("invalid issuance");
         let asset_id = AssetId::from_entropy(asset_entropy);
@@ -48,7 +50,7 @@ impl From<&TxIn> for IssuanceValue {
             contract_hash: contract_hash.map(|h| h.to_hex()),
             is_reissuance,
             asset_blinding_nonce: if is_reissuance {
-                Some(hex::encode(issuance.asset_blinding_nonce))
+                Some(hex::encode(issuance.asset_blinding_nonce.as_ref()))
             } else {
                 None
             },

src/elements/peg.rs

@@ -1,23 +1,40 @@
-use bitcoin::{hashes::hex::ToHex, Script};
+use elements::hex::ToHex;
 use elements::{confidential::Asset, PeginData, PegoutData, TxIn, TxOut};
 
-use crate::chain::Network;
-use crate::util::{get_script_asm, script_to_address, FullHash};
+use crate::chain::{bitcoin_genesis_hash, BNetwork, Network};
+use crate::util::FullHash;
 
-pub fn get_pegin_data(txout: &TxIn, network: Network) -> Option<PeginData> {
-    txout
-        .pegin_data()
-        .filter(|pegin| pegin.asset == Asset::Explicit(*network.native_asset()))
+pub fn get_pegin_data(txout: &TxIn, network: Network) -> Option<PeginData<'_>> {
+    let pegged_asset_id = network.pegged_asset()?;
+    txout.pegin_data().and_then(|pegin| {
+        if pegin.asset == *pegged_asset_id {
+            Some(pegin)
+        } else {
+            None
+        }
+    })
 }
 
 pub fn get_pegout_data(
     txout: &TxOut,
     network: Network,
-    parent_network: Network,
-) -> Option<PegoutData> {
-    txout.pegout_data().filter(|pegout| {
-        pegout.asset == Asset::Explicit(*network.native_asset())
-            && pegout.genesis_hash == parent_network.genesis_hash()
+    parent_network: BNetwork,
+) -> Option<PegoutData<'_>> {
+    let pegged_asset_id = network.pegged_asset()?;
+    txout.pegout_data().and_then(|pegout| {
+        if pegout.asset == Asset::Explicit(*pegged_asset_id)
+            && pegout.genesis_hash
+                == bitcoin_genesis_hash(match parent_network {
+                    BNetwork::Bitcoin => Network::Liquid,
+                    BNetwork::Testnet | BNetwork::Testnet4 => Network::LiquidTestnet,
+                    BNetwork::Signet => return None,
+                    BNetwork::Regtest => Network::LiquidRegtest,
+                })
+        {
+            Some(pegout)
+        } else {
+            None
+        }
     })
 }
 
@@ -25,37 +42,43 @@ pub fn get_pegout_data(
 #[derive(Serialize, Deserialize, Clone)]
 pub struct PegoutValue {
     pub genesis_hash: String,
-    pub scriptpubkey: Script,
+    pub scriptpubkey: bitcoin::ScriptBuf,
     pub scriptpubkey_asm: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub scriptpubkey_address: Option<String>,
 }
 
 impl PegoutValue {
-    pub fn from_txout(txout: &TxOut, network: Network, parent_network: Network) -> Option<Self> {
+    pub fn from_txout(txout: &TxOut, network: Network, parent_network: BNetwork) -> Option<Self> {
         let pegoutdata = get_pegout_data(txout, network, parent_network)?;
 
+        // pending https://github.com/ElementsProject/rust-elements/pull/69 is merged
+        let scriptpubkey = bitcoin::ScriptBuf::from(pegoutdata.script_pubkey.into_bytes());
+        let address = bitcoin::Address::from_script(&scriptpubkey, parent_network).ok();
+
         Some(PegoutValue {
             genesis_hash: pegoutdata.genesis_hash.to_hex(),
-            scriptpubkey_asm: get_script_asm(&pegoutdata.script_pubkey),
-            scriptpubkey_address: script_to_address(&pegoutdata.script_pubkey, parent_network),
-            scriptpubkey: pegoutdata.script_pubkey,
+            scriptpubkey_asm: scriptpubkey.to_asm_string(),
+            scriptpubkey_address: address.map(|s| s.to_string()),
+            scriptpubkey,
         })
     }
 }
 
 // Inner type for the indexer TxHistoryInfo::Pegin variant
 #[derive(Serialize, Deserialize, Debug)]
+#[cfg_attr(test, derive(PartialEq, Eq))]
 pub struct PeginInfo {
     pub txid: FullHash,
-    pub vin: u16,
+    pub vin: u32,
     pub value: u64,
 }
 
 // Inner type for the indexer TxHistoryInfo::Pegout variant
 #[derive(Serialize, Deserialize, Debug)]
+#[cfg_attr(test, derive(PartialEq, Eq))]
 pub struct PegoutInfo {
     pub txid: FullHash,
-    pub vout: u16,
+    pub vout: u32,
     pub value: u64,
 }

src/elements/registry.rs

@@ -1,32 +1,149 @@
-use std::fs;
-use std::path;
+use std::collections::HashMap;
+use std::str::FromStr;
+use std::sync::{Arc, RwLock};
+use std::time::{Duration, SystemTime};
+use std::{cmp, fs, path, thread};
 
 use serde_json::Value as JsonValue;
 
+use elements::AssetId;
+
 use crate::errors::*;
 
 // length of asset id prefix to use for sub-directory partitioning
 // (in number of hex characters, not bytes)
 
 const DIR_PARTITION_LEN: usize = 2;
+const SEARCH_SORT_CANDIDATE_LIMIT: usize = 2000;
+
 pub struct AssetRegistry {
     directory: path::PathBuf,
+    assets_cache: HashMap<AssetId, (SystemTime, AssetMeta)>,
 }
 
+pub type AssetEntry<'a> = (&'a AssetId, &'a AssetMeta);
+
 impl AssetRegistry {
     pub fn new(directory: path::PathBuf) -> Self {
-        Self { directory }
+        Self {
+            directory,
+            assets_cache: Default::default(),
+        }
     }
 
-    pub fn load<A: ToString>(&self, asset_id: A) -> Result<Option<AssetMeta>> {
-        let name = format!("{}.json", asset_id.to_string());
-        let subdir = self.directory.join(&name[0..DIR_PARTITION_LEN]);
-        let path = subdir.join(name);
-        Ok(if path.exists() {
-            let contents = fs::read_to_string(path).chain_err(|| "failed reading file")?;
-            Some(serde_json::from_str(&contents).chain_err(|| "failed parsing file")?)
-        } else {
-            None
+    pub fn get(&self, asset_id: &AssetId) -> Option<&AssetMeta> {
+        self.assets_cache
+            .get(asset_id)
+            .map(|(_, metadata)| metadata)
+    }
+
+    pub fn list(
+        &self,
+        start_index: usize,
+        limit: usize,
+        sorting: AssetSorting,
+    ) -> (usize, Vec<AssetEntry<'_>>) {
+        let mut assets: Vec<AssetEntry> = self
+            .assets_cache
+            .iter()
+            .map(|(asset_id, (_, metadata))| (asset_id, metadata))
+            .collect();
+        assets.sort_by(sorting.as_comparator());
+        (
+            assets.len(),
+            assets.into_iter().skip(start_index).take(limit).collect(),
+        )
+    }
+
+    pub fn search(&self, query: &str, limit: usize) -> Vec<AssetEntry<'_>> {
+        let query = query.trim();
+        if query.is_empty() || limit == 0 {
+            return vec![];
+        }
+
+        let (mut results, candidates) = search_by(
+            self.assets_cache
+                .iter()
+                .map(|(asset_id, (_, metadata))| (asset_id, metadata)),
+            query,
+            limit,
+            |metadata| metadata.ticker.as_deref(),
+        );
+
+        if results.len() < limit {
+            let (name_matches, candidates) =
+                search_by(candidates, query, limit - results.len(), |metadata| {
+                    Some(&metadata.name)
+                });
+            results.extend(name_matches);
+
+            if results.len() < limit {
+                let (domain_matches, _) =
+                    search_by(candidates, query, limit - results.len(), AssetMeta::domain);
+                results.extend(domain_matches);
+            }
+        }
+
+        results.truncate(limit);
+        results
+    }
+
+    pub fn fs_sync(&mut self) -> Result<()> {
+        for entry in fs::read_dir(&self.directory).chain_err(|| "failed reading asset dir")? {
+            let entry = entry.chain_err(|| "invalid fh")?;
+            let filetype = entry.file_type().chain_err(|| "failed getting file type")?;
+            if !filetype.is_dir() || entry.file_name().len() != DIR_PARTITION_LEN {
+                continue;
+            }
+
+            for file_entry in
+                fs::read_dir(entry.path()).chain_err(|| "failed reading asset subdir")?
+            {
+                let file_entry = file_entry.chain_err(|| "invalid fh")?;
+                let path = file_entry.path();
+                if path.extension().and_then(|e| e.to_str()) != Some("json") {
+                    continue;
+                }
+
+                let asset_id = AssetId::from_str(
+                    path.file_stem()
+                        .unwrap() // cannot fail if extension() succeeded
+                        .to_str()
+                        .chain_err(|| "invalid filename")?,
+                )
+                .chain_err(|| "invalid filename")?;
+
+                let modified = file_entry
+                    .metadata()
+                    .chain_err(|| "failed reading metadata")?
+                    .modified()
+                    .chain_err(|| "metadata modified failed")?;
+
+                if let Some((last_update, _)) = self.assets_cache.get(&asset_id) {
+                    if *last_update == modified {
+                        continue;
+                    }
+                }
+
+                let metadata: AssetMeta = serde_json::from_str(
+                    &fs::read_to_string(path).chain_err(|| "failed reading file")?,
+                )
+                .chain_err(|| "failed parsing file")?;
+
+                self.assets_cache.insert(asset_id, (modified, metadata));
+            }
+        }
+        Ok(())
+    }
+
+    pub fn spawn_sync(asset_db: Arc<RwLock<AssetRegistry>>) -> thread::JoinHandle<()> {
+        crate::util::spawn_thread("asset-registry", move || loop {
+            if let Err(e) = asset_db.write().unwrap().fs_sync() {
+                error!("registry fs_sync failed: {:?}", e);
+            }
+
+            thread::sleep(Duration::from_secs(15));
+            // TODO handle shutdowm
         })
     }
 }
@@ -42,3 +159,140 @@ pub struct AssetMeta {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ticker: Option<String>,
 }
+
+impl AssetMeta {
+    pub(crate) fn domain(&self) -> Option<&str> {
+        self.entity["domain"].as_str()
+    }
+}
+
+pub struct AssetSorting(AssetSortField, AssetSortDir);
+
+pub enum AssetSortField {
+    Name,
+    Domain,
+    Ticker,
+}
+pub enum AssetSortDir {
+    Descending,
+    Ascending,
+}
+
+type Comparator = Box<dyn Fn(&AssetEntry, &AssetEntry) -> cmp::Ordering>;
+
+impl AssetSorting {
+    #[allow(clippy::wrong_self_convention)]
+    fn as_comparator(self) -> Comparator {
+        let sort_fn: Comparator = match self.0 {
+            AssetSortField::Name => {
+                // Order by name first, use asset id as a tie breaker. the other sorting fields
+                // don't require this because they're guaranteed to be unique.
+                Box::new(|a, b| lc_cmp(&a.1.name, &b.1.name).then_with(|| a.0.cmp(b.0)))
+            }
+            AssetSortField::Domain => Box::new(|a, b| a.1.domain().cmp(&b.1.domain())),
+            AssetSortField::Ticker => Box::new(|a, b| lc_cmp_opt(&a.1.ticker, &b.1.ticker)),
+        };
+
+        match self.1 {
+            AssetSortDir::Ascending => sort_fn,
+            AssetSortDir::Descending => Box::new(move |a, b| sort_fn(a, b).reverse()),
+        }
+    }
+
+    pub fn from_query_params(query: &HashMap<String, String>) -> Result<Self> {
+        let field = match query.get("sort_field").map(String::as_str) {
+            None => AssetSortField::Ticker,
+            Some("name") => AssetSortField::Name,
+            Some("domain") => AssetSortField::Domain,
+            Some("ticker") => AssetSortField::Ticker,
+            _ => bail!("invalid sort field"),
+        };
+
+        let dir = match query.get("sort_dir").map(String::as_str) {
+            None => AssetSortDir::Ascending,
+            Some("asc") => AssetSortDir::Ascending,
+            Some("desc") => AssetSortDir::Descending,
+            _ => bail!("invalid sort direction"),
+        };
+
+        Ok(Self(field, dir))
+    }
+}
+
+fn lc_cmp(a: &str, b: &str) -> cmp::Ordering {
+    a.to_lowercase().cmp(&b.to_lowercase())
+}
+fn lc_cmp_opt(a: &Option<String>, b: &Option<String>) -> cmp::Ordering {
+    a.as_ref()
+        .map(|a| a.to_lowercase())
+        .cmp(&b.as_ref().map(|b| b.to_lowercase()))
+}
+
+fn search_by<'a, I, F>(
+    candidates: I,
+    query: &str,
+    limit: usize,
+    field: F,
+) -> (Vec<AssetEntry<'a>>, Vec<AssetEntry<'a>>)
+where
+    I: IntoIterator<Item = AssetEntry<'a>>,
+    F: Fn(&AssetMeta) -> Option<&str>,
+{
+    let mut matches = vec![];
+    let mut remaining = vec![];
+
+    for (asset_id, metadata) in candidates {
+        let position = field(metadata).and_then(|field| {
+            // registry fields are ascii, so we don't need full unicode case-folding
+            ascii_ci_find(field, query).map(|position| (position, field))
+        });
+
+        if let Some((position, field)) = position {
+            if matches.len() >= SEARCH_SORT_CANDIDATE_LIMIT {
+                continue;
+            }
+            matches.push((position, field, asset_id, metadata));
+        } else {
+            remaining.push((asset_id, metadata));
+        }
+    }
+
+    matches.sort_unstable_by(|a, b| {
+        a.0.cmp(&b.0)
+            .then_with(|| ascii_ci_cmp(a.1, b.1))
+            .then_with(|| a.2.cmp(b.2))
+    });
+
+    (
+        matches
+            .into_iter()
+            .take(limit)
+            .map(|(_, _, asset_id, metadata)| (asset_id, metadata))
+            .collect(),
+        remaining,
+    )
+}
+
+// zero-allocation case-insensitive ASCII substring search
+// returns the byte offset of the first match
+fn ascii_ci_find(haystack: &str, needle: &str) -> Option<usize> {
+    let (haystack, needle) = (haystack.as_bytes(), needle.as_bytes());
+    if needle.is_empty() {
+        return Some(0);
+    }
+    haystack
+        .windows(needle.len())
+        .position(|window| window.eq_ignore_ascii_case(needle))
+}
+
+// zero-allocation case-insensitive ASCII string comparison
+fn ascii_ci_cmp(a: &str, b: &str) -> cmp::Ordering {
+    let (a, b) = (a.as_bytes(), b.as_bytes());
+    for i in 0..a.len().min(b.len()) {
+        match a[i].to_ascii_lowercase().cmp(&b[i].to_ascii_lowercase()) {
+            cmp::Ordering::Equal => continue,
+            ord => return ord,
+        }
+    }
+    a.len().cmp(&b.len())
+}

src/errors.rs

@@ -1,3 +1,4 @@
+#![allow(unexpected_cfgs)]
 error_chain! {
     types {
         Error, ErrorKind, ResultExt, Result;
@@ -14,9 +15,14 @@ error_chain! {
             display("Iterrupted by signal {}", sig)
         }
 
-        TooPopular {
-            description("Too many history entries")
-            display("Too many history entries")
+        TooManyUtxos(limit: usize) {
+            description("Too many unspent transaction outputs. Contact support to raise limits.")
+            display("Too many unspent transaction outputs (>{}). Contact support to raise limits.", limit)
+        }
+
+        TooManyTxs(limit: usize) {
+            description("Too many history transactions. Contact support to raise limits.")
+            display("Too many history transactions (>{}). Contact support to raise limits.", limit)
         }
 
         #[cfg(feature = "electrum-discovery")]

src/lib.rs

@@ -1,10 +1,6 @@
 #![recursion_limit = "1024"]
 
-// See https://github.com/romanz/electrs/issues/193 & https://github.com/rust-rocksdb/rust-rocksdb/issues/327
-#[cfg(not(feature = "oldcpu"))]
 extern crate rocksdb;
-#[cfg(feature = "oldcpu")]
-extern crate rocksdb_oldcpu as rocksdb;
 
 #[macro_use]
 extern crate clap;

src/metrics.rs

@@ -5,7 +5,6 @@ use std::io;
 use std::net::SocketAddr;
 use std::thread;
 use std::time::Duration;
-use sysconf;
 use tiny_http;
 
 pub use prometheus::{
@@ -69,7 +68,7 @@ impl Metrics {
     pub fn start(&self) {
         let server = tiny_http::Server::http(self.addr)
             .unwrap_or_else(|_| panic!("failed to start monitoring HTTP server at {}", self.addr));
-        start_process_exporter(&self);
+        start_process_exporter(self);
         let reg = self.reg.clone();
         spawn_thread("metrics", move || loop {
             if let Err(e) = handle_request(&reg, server.recv()) {
@@ -98,6 +97,14 @@ struct Stats {
     fds: usize,
 }
 
+fn get_ticks_per_second() -> Result<f64> {
+    // Safety: This code is taken directly from sysconf
+    match unsafe { libc::sysconf(libc::_SC_CLK_TCK) } {
+        -1 => Err("Clock Tick unsupported".into()),
+        ret => Ok(ret as f64),
+    }
+}
+
 fn parse_stats() -> Result<Stats> {
     if cfg!(target_os = "macos") {
         return Ok(Stats {
@@ -109,15 +116,14 @@ fn parse_stats() -> Result<Stats> {
     let value = fs::read_to_string("/proc/self/stat").chain_err(|| "failed to read stats")?;
     let parts: Vec<&str> = value.split_whitespace().collect();
     let page_size = page_size::get() as u64;
-    let ticks_per_second = sysconf::raw::sysconf(sysconf::raw::SysconfVariable::ScClkTck)
-        .expect("failed to get _SC_CLK_TCK") as f64;
+    let ticks_per_second = get_ticks_per_second().expect("failed to get _SC_CLK_TCK");
 
     let parse_part = |index: usize, name: &str| -> Result<u64> {
-        Ok(parts
+        parts
             .get(index)
             .chain_err(|| format!("missing {}: {:?}", name, parts))?
             .parse::<u64>()
-            .chain_err(|| format!("invalid {}: {:?}", name, parts))?)
+            .chain_err(|| format!("invalid {}: {:?}", name, parts))
     };
 
     // For details, see '/proc/[pid]/stat' section at `man 5 proc`:
@@ -142,7 +148,7 @@ fn start_process_exporter(metrics: &Metrics) {
     spawn_thread("exporter", move || loop {
         match parse_stats() {
             Ok(stats) => {
-                cpu.with_label_values(&["utime"]).set(stats.utime as f64);
+                cpu.with_label_values(&["utime"]).set(stats.utime);
                 rss.set(stats.rss as i64);
                 fds.set(stats.fds as i64);
             }

src/new_index/db.rs

@@ -3,9 +3,13 @@ use rocksdb;
 use std::path::Path;
 
 use crate::config::Config;
-use crate::util::Bytes;
+use crate::util::{bincode_util, Bytes};
 
-static DB_VERSION: u32 = 1;
+/// Each version will break any running instance with a DB that has a differing version.
+/// It will also break if light mode is enabled or disabled.
+// 1 = Original DB (since fork from Blockstream)
+// 2 = Add tx position to TxHistory rows and place Spending before Funding
+static DB_VERSION: u32 = 2;
 
 #[derive(Debug, Eq, PartialEq)]
 pub struct DBRow {
@@ -19,14 +23,14 @@ pub struct ScanIterator<'a> {
     done: bool,
 }
 
-impl<'a> Iterator for ScanIterator<'a> {
+impl Iterator for ScanIterator<'_> {
     type Item = DBRow;
 
     fn next(&mut self) -> Option<DBRow> {
         if self.done {
             return None;
         }
-        let (key, value) = self.iter.next()?;
+        let (key, value) = self.iter.next().map(Result::ok)??;
         if !key.starts_with(&self.prefix) {
             self.done = true;
             return None;
@@ -44,7 +48,7 @@ pub struct ReverseScanIterator<'a> {
     done: bool,
 }
 
-impl<'a> Iterator for ReverseScanIterator<'a> {
+impl Iterator for ReverseScanIterator<'_> {
     type Item = DBRow;
 
     fn next(&mut self) -> Option<DBRow> {
@@ -69,6 +73,67 @@ impl<'a> Iterator for ReverseScanIterator<'a> {
     }
 }
 
+pub struct ReverseScanGroupIterator<'a> {
+    iters: Vec<ReverseScanIterator<'a>>,
+    next_rows: Vec<Option<DBRow>>,
+    value_offset: usize,
+    done: bool,
+}
+
+impl<'a> ReverseScanGroupIterator<'a> {
+    pub fn new(
+        mut iters: Vec<ReverseScanIterator<'a>>,
+        value_offset: usize,
+    ) -> ReverseScanGroupIterator<'a> {
+        let mut next_rows: Vec<Option<DBRow>> = Vec::with_capacity(iters.len());
+        for iter in &mut iters {
+            let next = iter.next();
+            next_rows.push(next);
+        }
+        let done = next_rows.iter().all(|row| row.is_none());
+        ReverseScanGroupIterator {
+            iters,
+            next_rows,
+            value_offset,
+            done,
+        }
+    }
+}
+
+impl Iterator for ReverseScanGroupIterator<'_> {
+    type Item = DBRow;
+
+    fn next(&mut self) -> Option<DBRow> {
+        if self.done {
+            return None;
+        }
+
+        let best_index = self
+            .next_rows
+            .iter()
+            .enumerate()
+            .max_by(|(a_index, a_opt), (b_index, b_opt)| match (a_opt, b_opt) {
+                (None, None) => a_index.cmp(b_index),
+
+                (Some(_), None) => std::cmp::Ordering::Greater,
+
+                (None, Some(_)) => std::cmp::Ordering::Less,
+
+                (Some(a), Some(b)) => a.key[self.value_offset..].cmp(&(b.key[self.value_offset..])),
+            })
+            .map(|(index, _)| index)
+            .unwrap_or(0);
+
+        let best = self.next_rows[best_index].take();
+        self.next_rows[best_index] = self.iters.get_mut(best_index)?.next();
+        if self.next_rows.iter().all(|row| row.is_none()) {
+            self.done = true;
+        }
+
+        best
+    }
+}
+
 #[derive(Debug)]
 pub struct DB {
     db: rocksdb::DB,
@@ -82,25 +147,8 @@ pub enum DBFlush {
 
 impl DB {
     pub fn open(path: &Path, config: &Config) -> DB {
-        debug!("opening DB at {:?}", path);
-        let mut db_opts = rocksdb::Options::default();
-        db_opts.create_if_missing(true);
-        db_opts.set_max_open_files(100_000); // TODO: make sure to `ulimit -n` this process correctly
-        db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
-        db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy);
-        db_opts.set_target_file_size_base(1_073_741_824);
-        db_opts.set_write_buffer_size(256 << 20);
-        db_opts.set_disable_auto_compactions(true); // for initial bulk load
-
-        // db_opts.set_advise_random_on_open(???);
-        db_opts.set_compaction_readahead_size(1 << 20);
-        db_opts.increase_parallelism(2);
-
-        // let mut block_opts = rocksdb::BlockBasedOptions::default();
-        // block_opts.set_block_size(???);
-
         let db = DB {
-            db: rocksdb::DB::open(&db_opts, path).expect("failed to open RocksDB"),
+            db: open_raw_db(path, OpenMode::ReadWrite),
         };
         db.verify_compatibility(config);
         db
@@ -118,11 +166,11 @@ impl DB {
         self.db.set_options(&opts).unwrap();
     }
 
-    pub fn raw_iterator(&self) -> rocksdb::DBRawIterator {
+    pub fn raw_iterator(&self) -> rocksdb::DBRawIterator<'_> {
         self.db.raw_iterator()
     }
 
-    pub fn iter_scan(&self, prefix: &[u8]) -> ScanIterator {
+    pub fn iter_scan(&self, prefix: &[u8]) -> ScanIterator<'_> {
         ScanIterator {
             prefix: prefix.to_vec(),
             iter: self.db.prefix_iterator(prefix),
@@ -130,7 +178,7 @@ impl DB {
         }
     }
 
-    pub fn iter_scan_from(&self, prefix: &[u8], start_at: &[u8]) -> ScanIterator {
+    pub fn iter_scan_from(&self, prefix: &[u8], start_at: &[u8]) -> ScanIterator<'_> {
         let iter = self.db.iterator(rocksdb::IteratorMode::From(
             start_at,
             rocksdb::Direction::Forward,
@@ -142,7 +190,7 @@ impl DB {
         }
     }
 
-    pub fn iter_scan_reverse(&self, prefix: &[u8], prefix_max: &[u8]) -> ReverseScanIterator {
+    pub fn iter_scan_reverse(&self, prefix: &[u8], prefix_max: &[u8]) -> ReverseScanIterator<'_> {
         let mut iter = self.db.raw_iterator();
         iter.seek_for_prev(prefix_max);
 
@@ -153,6 +201,25 @@ impl DB {
         }
     }
 
+    pub fn iter_scan_group_reverse(
+        &self,
+        prefixes: impl Iterator<Item = (Vec<u8>, Vec<u8>)>,
+        value_offset: usize,
+    ) -> ReverseScanGroupIterator<'_> {
+        let iters = prefixes
+            .map(|(prefix, prefix_max)| {
+                let mut iter = self.db.raw_iterator();
+                iter.seek_for_prev(prefix_max);
+                ReverseScanIterator {
+                    prefix: prefix.to_vec(),
+                    iter,
+                    done: false,
+                }
+            })
+            .collect();
+        ReverseScanGroupIterator::new(iters, value_offset)
+    }
+
     pub fn write(&self, mut rows: Vec<DBRow>, flush: DBFlush) {
         debug!(
             "writing {} rows to {:?}, flush={:?}",
@@ -163,10 +230,7 @@ impl DB {
         rows.sort_unstable_by(|a, b| a.key.cmp(&b.key));
         let mut batch = rocksdb::WriteBatch::default();
         for row in rows {
-            #[cfg(not(feature = "oldcpu"))]
             batch.put(&row.key, &row.value);
-            #[cfg(feature = "oldcpu")]
-            batch.put(&row.key, &row.value).unwrap();
         }
         let do_flush = match flush {
             DBFlush::Enable => true,
@@ -178,6 +242,15 @@ impl DB {
         self.db.write_opt(batch, &opts).unwrap();
     }
 
+    pub fn delete(&self, keys: Vec<Vec<u8>>) {
+        debug!("deleting {} rows from {:?}", keys.len(), self.db);
+        for key in keys {
+            let _ = self.db.delete(key).inspect_err(|err| {
+                warn!("Error while deleting DB row: {err}");
+            });
+        }
+    }
+
     pub fn flush(&self) {
         self.db.flush().unwrap();
     }
@@ -197,7 +270,7 @@ impl DB {
     }
 
     fn verify_compatibility(&self, config: &Config) {
-        let mut compatibility_bytes = bincode::serialize(&DB_VERSION).unwrap();
+        let mut compatibility_bytes = bincode_util::serialize_little(&DB_VERSION).unwrap();
 
         if config.light_mode {
             // append a byte to indicate light_mode is enabled.
@@ -216,3 +289,42 @@ impl DB {
         }
     }
 }
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+#[repr(u8)]
+pub enum OpenMode {
+    ReadOnly,
+    ReadWrite,
+}
+
+pub fn open_raw_db<T: rocksdb::ThreadMode>(
+    path: &Path,
+    read_mode: OpenMode,
+) -> rocksdb::DBWithThreadMode<T> {
+    debug!("opening DB at {:?}", path);
+    let mut db_opts = rocksdb::Options::default();
+    db_opts.create_if_missing(true);
+    db_opts.set_max_open_files(100_000); // TODO: make sure to `ulimit -n` this process correctly
+    db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
+    db_opts.set_compression_type(rocksdb::DBCompressionType::None);
+    db_opts.set_target_file_size_base(1_073_741_824);
+    db_opts.set_write_buffer_size(256 << 20);
+    db_opts.set_disable_auto_compactions(true); // for initial bulk load
+
+    // db_opts.set_advise_random_on_open(???);
+    db_opts.set_compaction_readahead_size(1 << 20);
+    db_opts.increase_parallelism(2);
+
+    // let mut block_opts = rocksdb::BlockBasedOptions::default();
+    // block_opts.set_block_size(???);
+
+    match read_mode {
+        OpenMode::ReadOnly => {
+            rocksdb::DBWithThreadMode::<T>::open_for_read_only(&db_opts, path, false)
+                .expect("failed to open RocksDB (READ ONLY)")
+        }
+        OpenMode::ReadWrite => {
+            rocksdb::DBWithThreadMode::<T>::open(&db_opts, path).expect("failed to open RocksDB")
+        }
+    }
+}

src/new_index/fetch.rs

@@ -1,6 +1,3 @@
-use crate::chain::Block;
-
-use bitcoin::{BitcoinHash, BlockHash};
 use rayon::prelude::*;
 
 #[cfg(not(feature = "liquid"))]
@@ -10,11 +7,11 @@ use elements::encode::{deserialize, Decodable};
 
 use std::collections::HashMap;
 use std::fs;
-use std::io::{Cursor, Seek, SeekFrom};
+use std::io::Cursor;
 use std::path::PathBuf;
-use std::sync::mpsc::Receiver;
 use std::thread;
 
+use crate::chain::{Block, BlockHash, BlockSizeCompat};
 use crate::daemon::Daemon;
 use crate::errors::*;
 use crate::util::{spawn_thread, HeaderEntry, SyncChannel};
@@ -45,19 +42,70 @@ pub struct BlockEntry {
 
 type SizedBlock = (Block, u32);
 
+pub struct SequentialFetcher<T> {
+    fetcher: Box<dyn FnOnce() -> Vec<Vec<T>>>,
+}
+
+impl<T> SequentialFetcher<T> {
+    fn from<F: FnOnce() -> Vec<Vec<T>> + 'static>(pre_func: F) -> Self {
+        SequentialFetcher {
+            fetcher: Box::new(pre_func),
+        }
+    }
+
+    pub fn map<FN>(self, mut func: FN)
+    where
+        FN: FnMut(Vec<T>),
+    {
+        for item in (self.fetcher)() {
+            func(item);
+        }
+    }
+}
+
+pub fn bitcoind_sequential_fetcher(
+    daemon: &Daemon,
+    new_headers: Vec<HeaderEntry>,
+) -> Result<SequentialFetcher<BlockEntry>> {
+    let daemon = daemon.reconnect()?;
+    Ok(SequentialFetcher::from(move || {
+        new_headers
+            .chunks(100)
+            .map(|entries| {
+                let blockhashes: Vec<BlockHash> = entries.iter().map(|e| *e.hash()).collect();
+                let blocks = daemon
+                    .getblocks(&blockhashes)
+                    .expect("failed to get blocks from bitcoind");
+                assert_eq!(blocks.len(), entries.len());
+                let block_entries: Vec<BlockEntry> = blocks
+                    .into_iter()
+                    .zip(entries)
+                    .map(|(block, entry)| BlockEntry {
+                        entry: entry.clone(), // TODO: remove this clone()
+                        size: block.get_block_size() as u32,
+                        block,
+                    })
+                    .collect();
+                assert_eq!(block_entries.len(), entries.len());
+                block_entries
+            })
+            .collect()
+    }))
+}
+
 pub struct Fetcher<T> {
-    receiver: Receiver<T>,
+    receiver: crossbeam_channel::Receiver<T>,
     thread: thread::JoinHandle<()>,
 }
 
 impl<T> Fetcher<T> {
-    fn from(receiver: Receiver<T>, thread: thread::JoinHandle<()>) -> Self {
+    fn from(receiver: crossbeam_channel::Receiver<T>, thread: thread::JoinHandle<()>) -> Self {
         Fetcher { receiver, thread }
     }
 
     pub fn map<F>(self, mut func: F)
     where
-        F: FnMut(T) -> (),
+        F: FnMut(T),
     {
         for item in self.receiver {
             func(item);
@@ -90,7 +138,7 @@ fn bitcoind_fetcher(
                     .zip(entries)
                     .map(|(block, entry)| BlockEntry {
                         entry: entry.clone(), // TODO: remove this clone()
-                        size: block.get_size() as u32,
+                        size: block.get_block_size() as u32,
                         block,
                     })
                     .collect();
@@ -124,7 +172,7 @@ fn blkfiles_fetcher(
                 let block_entries: Vec<BlockEntry> = sizedblocks
                     .into_iter()
                     .filter_map(|(block, size)| {
-                        let blockhash = block.bitcoin_hash();
+                        let blockhash = block.block_hash();
                         entry_map
                             .remove(&blockhash)
                             .map(|entry| BlockEntry { block, entry, size })
@@ -152,14 +200,33 @@ fn blkfiles_fetcher(
 fn blkfiles_reader(blk_files: Vec<PathBuf>) -> Fetcher<Vec<u8>> {
     let chan = SyncChannel::new(1);
     let sender = chan.sender();
+    let xor_key = blk_files.first().and_then(|p| {
+        let xor_file = p
+            .parent()
+            .expect("blk.dat files must exist in a directory")
+            .join("xor.dat");
+        if xor_file.exists() {
+            Some(fs::read(xor_file).expect("xor.dat exists"))
+        } else {
+            None
+        }
+    });
 
     Fetcher::from(
         chan.into_receiver(),
         spawn_thread("blkfiles_reader", move || {
             for path in blk_files {
                 trace!("reading {:?}", path);
-                let blob = fs::read(&path)
+                let mut blob = fs::read(&path)
                     .unwrap_or_else(|e| panic!("failed to read {:?}: {:?}", path, e));
+
+                // If the xor.dat exists. Use it to decrypt the block files.
+                if let Some(xor_key) = &xor_key {
+                    for (&key, byte) in xor_key.iter().cycle().zip(blob.iter_mut()) {
+                        *byte ^= key;
+                    }
+                }
+
                 sender
                     .send(blob)
                     .unwrap_or_else(|_| panic!("failed to send {:?} contents", path));
@@ -190,47 +257,40 @@ fn parse_blocks(blob: Vec<u8>, magic: u32) -> Result<Vec<SizedBlock>> {
     let mut cursor = Cursor::new(&blob);
     let mut slices = vec![];
     let max_pos = blob.len() as u64;
+
     while cursor.position() < max_pos {
+        let offset = cursor.position();
         match u32::consensus_decode(&mut cursor) {
             Ok(value) => {
                 if magic != value {
-                    cursor
-                        .seek(SeekFrom::Current(-3))
-                        .expect("failed to seek back");
+                    cursor.set_position(offset + 1);
                     continue;
                 }
             }
             Err(_) => break, // EOF
         };
         let block_size = u32::consensus_decode(&mut cursor).chain_err(|| "no block size")?;
-        let start = cursor.position() as usize;
-        cursor
-            .seek(SeekFrom::Current(block_size as i64))
-            .chain_err(|| format!("seek {} failed", block_size))?;
-        let end = cursor.position() as usize;
+        let start = cursor.position();
+        let end = start + block_size as u64;
 
-        // If Core's WriteBlockToDisk ftell fails, only the magic byte and size will be written
-        // and the block body will be unwritten data. skip that's data.
-        let mut tmp_cursor = Cursor::new(&blob[start..(start + 4)]);
-        match u32::consensus_decode(&mut tmp_cursor) {
+        // If Core's WriteBlockToDisk ftell fails, only the magic bytes and size will be written
+        // and the block body won't be written to the blk*.dat file.
+        // Since the first 4 bytes should contain the block's version, we can skip such blocks
+        // by peeking the cursor (and skipping previous `magic` and `block_size`).
+        match u32::consensus_decode(&mut cursor) {
             Ok(value) => {
                 if magic == value {
-                    cursor.set_position(start as u64);
+                    cursor.set_position(start);
                     continue;
                 }
             }
             Err(_) => break, // EOF
         }
-
-        slices.push((&blob[start..end], block_size));
+        slices.push((&blob[start as usize..end as usize], block_size));
+        cursor.set_position(end);
     }
 
-    let pool = rayon::ThreadPoolBuilder::new()
-        .num_threads(0) // CPU-bound
-        .thread_name(|i| format!("parse-blocks-{}", i))
-        .build()
-        .unwrap();
-    Ok(pool.install(|| {
+    Ok(super::THREAD_POOL.install(|| {
         slices
             .into_par_iter()
             .map(|(slice, size)| (deserialize(slice).expect("failed to parse Block"), size))

src/new_index/mempool.rs

@@ -1,6 +1,4 @@
-use arraydeque::{ArrayDeque, Wrapping};
-use bitcoin::consensus::encode::deserialize;
-use bitcoin::Txid;
+use bounded_vec_deque::BoundedVecDeque;
 use itertools::Itertools;
 
 #[cfg(not(feature = "liquid"))]
@@ -8,12 +6,13 @@ use bitcoin::consensus::encode::serialize;
 #[cfg(feature = "liquid")]
 use elements::{encode::serialize, AssetId};
 
-use std::collections::{BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::iter::FromIterator;
-use std::sync::Arc;
+use std::ops::Bound::{Excluded, Unbounded};
+use std::sync::{Arc, RwLock};
 use std::time::{Duration, Instant};
 
-use crate::chain::{Network, OutPoint, Transaction, TxOut};
+use crate::chain::{deserialize, Network, OutPoint, Transaction, TxOut, Txid, TxidCompat};
 use crate::config::Config;
 use crate::daemon::Daemon;
 use crate::errors::*;
@@ -28,17 +27,14 @@ use crate::util::{extract_tx_prevouts, full_hash, has_prevout, is_spendable, Byt
 #[cfg(feature = "liquid")]
 use crate::elements::asset;
 
-const RECENT_TXS_SIZE: usize = 10;
-const BACKLOG_STATS_TTL: u64 = 10;
-
 pub struct Mempool {
     chain: Arc<ChainQuery>,
     config: Arc<Config>,
-    txstore: HashMap<Txid, Transaction>,
+    txstore: BTreeMap<Txid, Transaction>,
     feeinfo: HashMap<Txid, TxFeeInfo>,
     history: HashMap<FullHash, Vec<TxHistoryInfo>>, // ScriptHash -> {history_entries}
     edges: HashMap<OutPoint, (Txid, u32)>,          // OutPoint -> (spending_txid, spending_vin)
-    recent: ArrayDeque<[TxOverview; RECENT_TXS_SIZE], Wrapping>, // The N most recent txs to enter the mempool
+    recent: BoundedVecDeque<TxOverview>,            // The N most recent txs to enter the mempool
     backlog_stats: (BacklogStats, Instant),
 
     // monitoring
@@ -67,15 +63,14 @@ impl Mempool {
     pub fn new(chain: Arc<ChainQuery>, metrics: &Metrics, config: Arc<Config>) -> Self {
         Mempool {
             chain,
-            config,
-            txstore: HashMap::new(),
+            txstore: BTreeMap::new(),
             feeinfo: HashMap::new(),
             history: HashMap::new(),
             edges: HashMap::new(),
-            recent: ArrayDeque::new(),
+            recent: BoundedVecDeque::new(config.mempool_recent_txs_size),
             backlog_stats: (
                 BacklogStats::default(),
-                Instant::now() - Duration::from_secs(BACKLOG_STATS_TTL),
+                Instant::now() - Duration::from_secs(config.mempool_backlog_stats_ttl),
             ),
             latency: metrics.histogram_vec(
                 HistogramOpts::new("mempool_latency", "Mempool requests latency (in seconds)"),
@@ -94,6 +89,7 @@ impl Mempool {
             asset_history: HashMap::new(),
             #[cfg(feature = "liquid")]
             asset_issuance: HashMap::new(),
+            config,
         }
     }
 
@@ -135,24 +131,95 @@ impl Mempool {
             .any(|txin| self.txstore.contains_key(&txin.previous_output.txid))
     }
 
-    pub fn history(&self, scripthash: &[u8], limit: usize) -> Vec<Transaction> {
+    pub fn history(
+        &self,
+        scripthash: &[u8],
+        last_seen_txid: Option<&Txid>,
+        limit: usize,
+    ) -> Vec<Transaction> {
         let _timer = self.latency.with_label_values(&["history"]).start_timer();
         self.history
             .get(scripthash)
-            .map_or_else(|| vec![], |entries| self._history(entries, limit))
+            .map_or_else(std::vec::Vec::new, |entries| {
+                self._history(entries, last_seen_txid, limit)
+            })
     }
 
-    fn _history(&self, entries: &[TxHistoryInfo], limit: usize) -> Vec<Transaction> {
+    pub fn history_txids_iter<'a>(&'a self, scripthash: &[u8]) -> impl Iterator<Item = Txid> + 'a {
+        self.history
+            .get(scripthash)
+            .into_iter()
+            .flat_map(|v| v.iter().map(|e| e.get_txid()).unique())
+    }
+
+    fn _history(
+        &self,
+        entries: &[TxHistoryInfo],
+        last_seen_txid: Option<&Txid>,
+        limit: usize,
+    ) -> Vec<Transaction> {
         entries
             .iter()
             .map(|e| e.get_txid())
             .unique()
+            // TODO seek directly to last seen tx without reading earlier rows
+            .skip_while(|txid| {
+                // skip until we reach the last_seen_txid
+                last_seen_txid.is_some_and(|last_seen_txid| last_seen_txid != txid)
+            })
+            .skip(match last_seen_txid {
+                Some(_) => 1, // skip the last_seen_txid itself
+                None => 0,
+            })
             .take(limit)
             .map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
             .cloned()
             .collect()
     }
 
+    pub fn history_group(
+        &self,
+        scripthashes: &[[u8; 32]],
+        last_seen_txid: Option<&Txid>,
+        limit: usize,
+    ) -> Vec<Transaction> {
+        let _timer = self
+            .latency
+            .with_label_values(&["history_group"])
+            .start_timer();
+        scripthashes
+            .iter()
+            .filter_map(|scripthash| self.history.get(&scripthash[..]))
+            .flat_map(|entries| entries.iter())
+            .map(|e| e.get_txid())
+            .unique()
+            // TODO seek directly to last seen tx without reading earlier rows
+            .skip_while(|txid| {
+                // skip until we reach the last_seen_txid
+                last_seen_txid.is_some_and(|last_seen_txid| last_seen_txid != txid)
+            })
+            .skip(match last_seen_txid {
+                Some(_) => 1, // skip the last_seen_txid itself
+                None => 0,
+            })
+            .take(limit)
+            .map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
+            .cloned()
+            .collect()
+    }
+
+    pub fn history_txids_iter_group<'a>(
+        &'a self,
+        scripthashes: &'a [[u8; 32]],
+    ) -> impl Iterator<Item = Txid> + 'a {
+        scripthashes
+            .iter()
+            .filter_map(move |scripthash| self.history.get(&scripthash[..]))
+            .flat_map(|entries| entries.iter())
+            .map(|entry| entry.get_txid())
+            .unique()
+    }
+
     pub fn history_txids(&self, scripthash: &[u8], limit: usize) -> Vec<Txid> {
         let _timer = self
             .latency
@@ -182,13 +249,11 @@ impl Mempool {
                 TxHistoryInfo::Funding(info) => {
                     // Liquid requires some additional information from the txo that's not available in the TxHistoryInfo index.
                     #[cfg(feature = "liquid")]
-                    let txo = self
-                        .lookup_txo(&entry.get_funded_outpoint())
-                        .expect("missing txo");
+                    let txo = self.lookup_txo(&entry.get_funded_outpoint())?;
 
                     Some(Utxo {
                         txid: deserialize(&info.txid).expect("invalid txid"),
-                        vout: info.vout as u32,
+                        vout: info.vout,
                         value: info.value,
                         confirmed: None,
 
@@ -266,6 +331,49 @@ impl Mempool {
         self.txstore.keys().collect()
     }
 
+    // Get n txids after the given txid in the mempool
+    pub fn txids_page(&self, n: usize, start: Option<Txid>) -> Vec<&Txid> {
+        let _timer = self
+            .latency
+            .with_label_values(&["txids_page"])
+            .start_timer();
+        let start_bound = match start {
+            Some(txid) => Excluded(txid),
+            None => Unbounded,
+        };
+
+        self.txstore
+            .range((start_bound, Unbounded))
+            .take(n)
+            .map(|(k, _v)| k)
+            .collect()
+    }
+
+    // Get all txs in the mempool
+    pub fn txs(&self) -> Vec<Transaction> {
+        let _timer = self.latency.with_label_values(&["txs"]).start_timer();
+        self.txstore.values().cloned().collect()
+    }
+
+    // Get n txs after the given txid in the mempool
+    pub fn txs_page(&self, n: usize, start: Option<Txid>) -> Vec<Transaction> {
+        let _timer = self.latency.with_label_values(&["txs_page"]).start_timer();
+        let mut page = Vec::with_capacity(n);
+        let start_bound = match start {
+            Some(txid) => Excluded(txid),
+            None => Unbounded,
+        };
+
+        self.txstore
+            .range((start_bound, Unbounded))
+            .take(n)
+            .for_each(|(_, value)| {
+                page.push(value.clone());
+            });
+
+        page
+    }
+
     // Get an overview of the most recent transactions
     pub fn recent_txs_overview(&self) -> Vec<&TxOverview> {
         // We don't bother ever deleting elements from the recent list.
@@ -278,87 +386,150 @@ impl Mempool {
         &self.backlog_stats.0
     }
 
-    pub fn update(&mut self, daemon: &Daemon) -> Result<()> {
-        let _timer = self.latency.with_label_values(&["update"]).start_timer();
-        let new_txids = daemon
+    pub fn unique_txids(&self) -> HashSet<Txid> {
+        HashSet::from_iter(self.txstore.keys().cloned())
+    }
+
+    pub fn update(mempool: &RwLock<Mempool>, daemon: &Daemon) -> Result<()> {
+        // 1. Start the metrics timer and get the current mempool txids
+        // [LOCK] Takes read lock for whole scope.
+        let (_timer, old_txids) = {
+            let mempool = mempool.read().unwrap();
+            (
+                mempool.latency.with_label_values(&["update"]).start_timer(),
+                mempool.unique_txids(),
+            )
+        };
+
+        // 2. Get all the mempool txids from the RPC.
+        // [LOCK] No lock taken. Wait for RPC request. Get lists of remove/add txes.
+        let all_txids = daemon
             .getmempooltxids()
             .chain_err(|| "failed to update mempool from daemon")?;
-        let old_txids = HashSet::from_iter(self.txstore.keys().cloned());
-        let to_remove: HashSet<&Txid> = old_txids.difference(&new_txids).collect();
+        let txids_to_remove: HashSet<&Txid> = old_txids.difference(&all_txids).collect();
+        let txids_to_add: Vec<&Txid> = all_txids.difference(&old_txids).collect();
 
-        // Download and add new transactions from bitcoind's mempool
-        let txids: Vec<&Txid> = new_txids.difference(&old_txids).collect();
-        let to_add = match daemon.gettransactions(&txids) {
-            Ok(txs) => txs,
-            Err(err) => {
-                warn!("failed to get transactions {:?}: {}", txids, err); // e.g. new block or RBF
-                return Ok(()); // keep the mempool until next update()
+        // 3. Remove missing transactions. Even if we are unable to download new transactions from
+        // the daemon, we still want to remove the transactions that are no longer in the mempool.
+        // [LOCK] Write lock is released at the end of the call to remove().
+        mempool.write().unwrap().remove(txids_to_remove);
+
+        // 4. Download the new transactions from the daemon's mempool
+        // [LOCK] No lock taken, waiting for RPC response.
+        let txs_to_add = daemon
+            .gettransactions(&txids_to_add)
+            .chain_err(|| format!("failed to get {} transactions", txids_to_add.len()))?;
+
+        // 4. Update local mempool to match daemon's state
+        // [LOCK] Takes Write lock for whole scope.
+        {
+            let mut mempool = mempool.write().unwrap();
+            // Add new transactions
+            if txs_to_add.len() > mempool.add(txs_to_add) {
+                debug!("Mempool update added less transactions than expected");
             }
-        };
-        // Add new transactions
-        self.add(to_add);
-        // Remove missing transactions
-        self.remove(to_remove);
 
-        self.count
-            .with_label_values(&["txs"])
-            .set(self.txstore.len() as f64);
+            mempool
+                .count
+                .with_label_values(&["txs"])
+                .set(mempool.txstore.len() as f64);
 
-        // Update cached backlog stats (if expired)
-        if self.backlog_stats.1.elapsed() > Duration::from_secs(BACKLOG_STATS_TTL) {
-            let _timer = self
-                .latency
-                .with_label_values(&["update_backlog_stats"])
-                .start_timer();
-            self.backlog_stats = (BacklogStats::new(&self.feeinfo), Instant::now());
+            // Update cached backlog stats (if expired)
+            if mempool.backlog_stats.1.elapsed()
+                > Duration::from_secs(mempool.config.mempool_backlog_stats_ttl)
+            {
+                let _timer = mempool
+                    .latency
+                    .with_label_values(&["update_backlog_stats"])
+                    .start_timer();
+                mempool.backlog_stats = (BacklogStats::new(&mempool.feeinfo), Instant::now());
+            }
+
+            Ok(())
         }
+    }
 
+    pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Txid) -> Result<()> {
+        if !self.txstore.contains_key(txid) {
+            if let Ok(tx) = daemon.getmempooltx(txid) {
+                if self.add(vec![tx]) == 0 {
+                    return Err(format!(
+                        "Unable to add {txid} to mempool likely due to missing parents."
+                    )
+                    .into());
+                }
+            }
+        }
         Ok(())
     }
 
-    pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Txid) {
-        if let Ok(tx) = daemon.getmempooltx(&txid) {
-            self.add(vec![tx])
-        }
-    }
-
-    fn add(&mut self, txs: Vec<Transaction>) {
+    /// Add transactions to the mempool.
+    ///
+    /// The return value is the number of transactions processed.
+    #[must_use = "Must deal with [[input vec's length]] > [[result]]."]
+    fn add(&mut self, txs: Vec<Transaction>) -> usize {
         self.delta
             .with_label_values(&["add"])
             .observe(txs.len() as f64);
         let _timer = self.latency.with_label_values(&["add"]).start_timer();
+        let txlen = txs.len();
+        if txlen == 0 {
+            return 0;
+        }
+        debug!("Adding {} transactions to Mempool", txlen);
 
-        let mut txids = vec![];
+        let mut txids = Vec::with_capacity(txs.len());
         // Phase 1: add to txstore
         for tx in txs {
-            let txid = tx.txid();
-            txids.push(txid);
-            self.txstore.insert(txid, tx);
-        }
-        // Phase 2: index history and spend edges (can fail if some txos cannot be found)
-        let txos = match self.lookup_txos(&self.get_prevouts(&txids)) {
-            Ok(txos) => txos,
-            Err(err) => {
-                warn!("lookup txouts failed: {}", err);
-                // TODO: should we remove txids from txstore?
-                return;
+            let txid = tx.get_txid();
+            // Only push if it doesn't already exist.
+            // This is important now that update doesn't lock during
+            // the entire function body.
+            if self.txstore.insert(txid, tx).is_none() {
+                txids.push(txid);
             }
-        };
+        }
+
+        // Phase 2: index history and spend edges (some txos can be missing)
+        let txos = self.lookup_txos(&self.get_prevouts(&txids));
+
+        // Count how many transactions were actually processed.
+        let mut processed_count = 0;
+
+        // Phase 3: Iterate over the transactions and do the following:
+        // 1. Find all of the TxOuts of each input parent using `txos`
+        // 2. If any parent wasn't found, skip parsing this transaction
+        // 3. Insert TxFeeInfo into info.
+        // 4. Push TxOverview into recent tx queue.
+        // 5. Create the Spend and Fund TxHistory structs for inputs + outputs
+        // 6. Insert all TxHistory into history.
+        // 7. Insert the tx edges into edges (HashMap of (Outpoint, (Txid, vin)))
+        // 8. (Liquid only) Parse assets of tx.
         for txid in txids {
-            let tx = self.txstore.get(&txid).expect("missing mempool tx");
+            let tx = self.txstore.get(&txid).expect("missing tx from txstore");
+
+            let prevouts = match extract_tx_prevouts(tx, &txos) {
+                Ok(v) => v,
+                Err(e) => {
+                    warn!("Skipping tx {txid} missing parent error: {e}");
+                    continue;
+                }
+            };
             let txid_bytes = full_hash(&txid[..]);
-            let prevouts = extract_tx_prevouts(&tx, &txos, false);
 
             // Get feeinfo for caching and recent tx overview
-            let feeinfo = TxFeeInfo::new(&tx, &prevouts, self.config.network_type);
+            let feeinfo = TxFeeInfo::new(tx, &prevouts, self.config.network_type);
 
-            // recent is an ArrayDeque that automatically evicts the oldest elements
+            // recent is an BoundedVecDeque that automatically evicts the oldest elements
             self.recent.push_front(TxOverview {
                 txid,
                 fee: feeinfo.fee,
                 vsize: feeinfo.vsize,
                 #[cfg(not(feature = "liquid"))]
-                value: prevouts.values().map(|prevout| prevout.value).sum(),
+                value: prevouts
+                    .values()
+                    .map(|prevout| prevout.value.to_sat())
+                    .sum(),
             });
 
             self.feeinfo.insert(txid, feeinfo);
@@ -366,41 +537,48 @@ impl Mempool {
             // An iterator over (ScriptHash, TxHistoryInfo)
             let spending = prevouts.into_iter().map(|(input_index, prevout)| {
                 let txi = tx.input.get(input_index as usize).unwrap();
+                #[cfg(not(feature = "liquid"))]
+                let value = prevout.value.to_sat();
+                #[cfg(feature = "liquid")]
+                let value = prevout.value;
                 (
                     compute_script_hash(&prevout.script_pubkey),
                     TxHistoryInfo::Spending(SpendingInfo {
                         txid: txid_bytes,
-                        vin: input_index as u16,
+                        vin: input_index,
                         prev_txid: full_hash(&txi.previous_output.txid[..]),
-                        prev_vout: txi.previous_output.vout as u16,
-                        value: prevout.value,
+                        prev_vout: txi.previous_output.vout,
+                        value,
                     }),
                 )
             });
 
+            let config = &self.config;
+
             // An iterator over (ScriptHash, TxHistoryInfo)
             let funding = tx
                 .output
                 .iter()
                 .enumerate()
-                .filter(|(_, txo)| is_spendable(txo))
+                .filter(|(_, txo)| is_spendable(txo) || config.index_unspendables)
                 .map(|(index, txo)| {
+                    #[cfg(not(feature = "liquid"))]
+                    let value = txo.value.to_sat();
+                    #[cfg(feature = "liquid")]
+                    let value = txo.value;
                     (
                         compute_script_hash(&txo.script_pubkey),
                         TxHistoryInfo::Funding(FundingInfo {
                             txid: txid_bytes,
-                            vout: index as u16,
-                            value: txo.value,
+                            vout: index as u32,
+                            value,
                         }),
                     )
                 });
 
             // Index funding/spending history entries and spend edges
             for (scripthash, entry) in funding.chain(spending) {
-                self.history
-                    .entry(scripthash)
-                    .or_insert_with(Vec::new)
-                    .push(entry);
+                self.history.entry(scripthash).or_default().push(entry);
             }
             for (i, txi) in tx.input.iter().enumerate() {
                 self.edges.insert(txi.previous_output, (txid, i as u32));
@@ -409,22 +587,32 @@ impl Mempool {
             // Index issued assets & native asset pegins/pegouts/burns
             #[cfg(feature = "liquid")]
             asset::index_mempool_tx_assets(
-                &tx,
+                tx,
                 self.config.network_type,
                 self.config.parent_network,
                 &mut self.asset_history,
                 &mut self.asset_issuance,
             );
+
+            processed_count += 1;
         }
+
+        processed_count
     }
 
-    pub fn lookup_txo(&self, outpoint: &OutPoint) -> Result<TxOut> {
+    /// Returns None if the lookup fails (mempool transaction RBF-ed etc.)
+    pub fn lookup_txo(&self, outpoint: &OutPoint) -> Option<TxOut> {
         let mut outpoints = BTreeSet::new();
         outpoints.insert(*outpoint);
-        Ok(self.lookup_txos(&outpoints)?.remove(outpoint).unwrap())
+        // This can possibly be None now
+        self.lookup_txos(&outpoints).remove(outpoint)
     }
 
-    pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> Result<HashMap<OutPoint, TxOut>> {
+    /// For a given set of OutPoints, return a HashMap<OutPoint, TxOut>
+    ///
+    /// Not all OutPoints from mempool transactions are guaranteed to be there.
+    /// Ensure you deal with the None case in your logic.
+    pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
         let _timer = self
             .latency
             .with_label_values(&["lookup_txos"])
@@ -435,18 +623,21 @@ impl Mempool {
         let mempool_txos = outpoints
             .iter()
             .filter(|outpoint| !confirmed_txos.contains_key(outpoint))
-            .map(|outpoint| {
+            .flat_map(|outpoint| {
                 self.txstore
                     .get(&outpoint.txid)
                     .and_then(|tx| tx.output.get(outpoint.vout as usize).cloned())
                     .map(|txout| (*outpoint, txout))
-                    .chain_err(|| format!("missing outpoint {:?}", outpoint))
+                    .or_else(|| {
+                        warn!("missing outpoint {:?}", outpoint);
+                        None
+                    })
             })
-            .collect::<Result<HashMap<OutPoint, TxOut>>>()?;
+            .collect::<HashMap<OutPoint, TxOut>>();
 
         let mut txos = confirmed_txos;
         txos.extend(mempool_txos);
-        Ok(txos)
+        txos
     }
 
     fn get_prevouts(&self, txids: &[Txid]) -> BTreeSet<OutPoint> {
@@ -509,7 +700,9 @@ impl Mempool {
             .start_timer();
         self.asset_history
             .get(asset_id)
-            .map_or_else(|| vec![], |entries| self._history(entries, limit))
+            .map_or_else(std::vec::Vec::new, |entries| {
+                self._history(entries, None, limit)
+            })
     }
 }
 

src/new_index/mod.rs

@@ -5,6 +5,16 @@ pub mod precache;
 mod query;
 pub mod schema;
 
+use std::sync::LazyLock;
+
+pub(crate) static THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(0) // 0 = use number of logical CPUs
+        .thread_name(|i| format!("electrs-worker-{}", i))
+        .build()
+        .expect("failed to create global rayon thread pool")
+});
+
 pub use self::db::{DBRow, DB};
 pub use self::fetch::{BlockEntry, FetchFrom};
 pub use self::mempool::Mempool;

src/new_index/precache.rs

@@ -1,38 +1,59 @@
-use crate::chain::address::Address;
 use crate::errors::*;
 use crate::new_index::ChainQuery;
 use crate::util::{full_hash, FullHash};
 
-use crypto::digest::Digest;
-use crypto::sha2::Sha256;
 use rayon::prelude::*;
 
 use hex;
 use std::fs::File;
 use std::io;
 use std::io::prelude::*;
-use std::str::FromStr;
+use std::sync::{atomic::AtomicUsize, Arc};
+use std::time::Instant;
 
-pub fn precache(chain: &ChainQuery, scripthashes: Vec<FullHash>) {
+pub fn precache(chain: Arc<ChainQuery>, scripthashes: Vec<FullHash>, threads: usize) {
     let total = scripthashes.len();
-    info!("Pre-caching stats and utxo set for {} scripthashes", total);
+    info!(
+        "Pre-caching stats and utxo set on {} threads for {} scripthashes",
+        threads, total
+    );
 
     let pool = rayon::ThreadPoolBuilder::new()
-        .num_threads(16)
+        .num_threads(threads)
         .thread_name(|i| format!("precache-{}", i))
         .build()
         .unwrap();
-    pool.install(|| {
-        scripthashes
-            .par_iter()
-            .enumerate()
-            .for_each(|(i, scripthash)| {
-                if i % 5 == 0 {
-                    info!("running pre-cache for scripthash {}/{}", i + 1, total);
-                }
-                chain.stats(&scripthash[..]);
-                //chain.utxo(&scripthash[..]);
-            })
+    let now = Instant::now();
+    let counter = AtomicUsize::new(0);
+    std::thread::spawn(move || {
+        pool.install(|| {
+            scripthashes
+                .par_iter()
+                .for_each(|scripthash| {
+                    // First, cache
+                    chain.stats(&scripthash[..], crate::new_index::db::DBFlush::Disable);
+                    let _ = chain.utxo(&scripthash[..], usize::MAX, crate::new_index::db::DBFlush::Disable);
+
+                    // Then, increment the counter
+                    let pre_increment = counter.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                    let post_increment_counter = pre_increment + 1;
+
+                    // Then, log
+                    if post_increment_counter % 500 == 0 {
+                        let now_millis = now.elapsed().as_millis();
+                        info!("{post_increment_counter}/{total} Processed in {now_millis} ms running pre-cache for scripthash");
+                    }
+
+                    // Every 10k counts, flush the DB to disk
+                    if post_increment_counter % 10000 == 0 {
+                        info!("Flushing cache_db... {post_increment_counter}");
+                        chain.store().cache_db().flush();
+                        info!("Done Flushing cache_db!!! {post_increment_counter}");
+                    }
+                })
+        });
+        // After everything is done, flush the cache
+        chain.store().cache_db().flush();
     });
 }
 
@@ -43,34 +64,7 @@ pub fn scripthashes_from_file(path: String) -> Result<Vec<FullHash>> {
         .lines()
         .map(|line| {
             let line = line.chain_err(|| "cannot read scripthash line")?;
-            let cols: Vec<&str> = line.split(',').collect();
-            to_scripthash(cols[0], cols[1])
+            Ok(full_hash(&hex::decode(line).chain_err(|| "invalid hex")?))
         })
         .collect()
 }
-
-fn to_scripthash(script_type: &str, script_str: &str) -> Result<FullHash> {
-    match script_type {
-        "address" => address_to_scripthash(script_str),
-        "scripthash" => Ok(full_hash(
-            &hex::decode(script_str).chain_err(|| "invalid hex")?,
-        )),
-        "scriptpubkey" => Ok(compute_script_hash(
-            &hex::decode(script_str).chain_err(|| "invalid hex")?,
-        )),
-        _ => bail!("Invalid script type".to_string()),
-    }
-}
-
-fn address_to_scripthash(addr: &str) -> Result<FullHash> {
-    let addr = Address::from_str(addr).chain_err(|| "invalid address")?;
-    Ok(compute_script_hash(&addr.script_pubkey().as_bytes()))
-}
-
-pub fn compute_script_hash(data: &[u8]) -> FullHash {
-    let mut hash = FullHash::default();
-    let mut sha2 = Sha256::new();
-    sha2.input(data);
-    sha2.result(&mut hash);
-    hash
-}

src/new_index/query.rs

@@ -4,19 +4,17 @@ use std::collections::{BTreeSet, HashMap};
 use std::sync::{Arc, RwLock, RwLockReadGuard};
 use std::time::{Duration, Instant};
 
-use crate::chain::{Network, OutPoint, Transaction, TxOut};
+use crate::chain::{Network, OutPoint, Transaction, TxOut, Txid, TxidCompat};
 use crate::config::Config;
-use crate::daemon::Daemon;
+use crate::daemon::{Daemon, MempoolAcceptResult, SubmitPackageResult};
 use crate::errors::*;
 use crate::new_index::{ChainQuery, Mempool, ScriptStats, SpendingInput, Utxo};
 use crate::util::{is_spendable, BlockId, Bytes, TransactionStatus};
 
-use bitcoin::Txid;
-
 #[cfg(feature = "liquid")]
 use crate::{
-    chain::AssetId,
-    elements::{lookup_asset, AssetRegistry, LiquidAsset},
+    chain::{asset::AssetRegistryLock, AssetId},
+    elements::{lookup_asset, AssetMeta, AssetRegistry, AssetSorting, LiquidAsset},
 };
 
 const FEE_ESTIMATES_TTL: u64 = 60; // seconds
@@ -34,7 +32,7 @@ pub struct Query {
     cached_estimates: RwLock<(HashMap<u16, f64>, Option<Instant>)>,
     cached_relayfee: RwLock<Option<f64>>,
     #[cfg(feature = "liquid")]
-    asset_db: Option<AssetRegistry>,
+    asset_db: Option<Arc<RwLock<AssetRegistry>>>,
 }
 
 impl Query {
@@ -67,21 +65,51 @@ impl Query {
         self.config.network_type
     }
 
-    pub fn mempool(&self) -> RwLockReadGuard<Mempool> {
+    pub fn mempool(&self) -> RwLockReadGuard<'_, Mempool> {
         self.mempool.read().unwrap()
     }
 
     pub fn broadcast_raw(&self, txhex: &str) -> Result<Txid> {
         let txid = self.daemon.broadcast_raw(txhex)?;
-        self.mempool
+        // The important part is whether we succeeded in broadcasting.
+        // Ignore errors in adding to the cache and show an internal warning.
+        if let Err(e) = self
+            .mempool
             .write()
             .unwrap()
-            .add_by_txid(&self.daemon, &txid);
+            .add_by_txid(&self.daemon, &txid)
+        {
+            warn!(
+                "broadcast_raw of {txid} succeeded to broadcast \
+                but failed to add to mempool-electrs Mempool cache: {e}"
+            );
+        }
         Ok(txid)
     }
 
+    pub fn test_mempool_accept(
+        &self,
+        txhex: Vec<String>,
+        maxfeerate: Option<f64>,
+    ) -> Result<Vec<MempoolAcceptResult>> {
+        self.daemon.test_mempool_accept(txhex, maxfeerate)
+    }
+
+    pub fn submit_package(
+        &self,
+        txhex: Vec<String>,
+        maxfeerate: Option<f64>,
+        maxburnamount: Option<f64>,
+    ) -> Result<SubmitPackageResult> {
+        self.daemon.submit_package(txhex, maxfeerate, maxburnamount)
+    }
+
     pub fn utxo(&self, scripthash: &[u8]) -> Result<Vec<Utxo>> {
-        let mut utxos = self.chain.utxo(scripthash, self.config.utxos_limit)?;
+        let mut utxos = self.chain.utxo(
+            scripthash,
+            self.config.utxos_limit,
+            super::db::DBFlush::Enable,
+        )?;
         let mempool = self.mempool();
         utxos.retain(|utxo| !mempool.has_spend(&OutPoint::from(utxo)));
         utxos.extend(mempool.utxo(scripthash));
@@ -104,7 +132,7 @@ impl Query {
 
     pub fn stats(&self, scripthash: &[u8]) -> (ScriptStats, ScriptStats) {
         (
-            self.chain.stats(scripthash),
+            self.chain.stats(scripthash, super::db::DBFlush::Enable),
             self.mempool().stats(scripthash),
         )
     }
@@ -120,11 +148,10 @@ impl Query {
             .or_else(|| self.mempool().lookup_raw_txn(txid))
     }
 
+    /// Not all OutPoints from mempool transactions are guaranteed to be included in the result
     pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
         // the mempool lookup_txos() internally looks up confirmed txos as well
-        self.mempool()
-            .lookup_txos(outpoints)
-            .expect("failed loading txos")
+        self.mempool().lookup_txos(outpoints)
     }
 
     pub fn lookup_spend(&self, outpoint: &OutPoint) -> Option<SpendingInput> {
@@ -134,7 +161,7 @@ impl Query {
     }
 
     pub fn lookup_tx_spends(&self, tx: Transaction) -> Vec<Option<SpendingInput>> {
-        let txid = tx.txid();
+        let txid = tx.get_txid();
 
         tx.output
             .par_iter()
@@ -165,7 +192,7 @@ impl Query {
     }
 
     pub fn estimate_fee(&self, conf_target: u16) -> Option<f64> {
-        if self.config.network_type == Network::Regtest {
+        if self.config.network_type.is_regtest() {
             return self.get_relayfee().ok();
         }
         if let (ref cache, Some(cache_time)) = *self.cached_estimates.read().unwrap() {
@@ -221,7 +248,7 @@ impl Query {
         mempool: Arc<RwLock<Mempool>>,
         daemon: Arc<Daemon>,
         config: Arc<Config>,
-        asset_db: Option<AssetRegistry>,
+        asset_db: Option<Arc<RwLock<AssetRegistry>>>,
     ) -> Self {
         Query {
             chain,
@@ -236,6 +263,71 @@ impl Query {
 
     #[cfg(feature = "liquid")]
     pub fn lookup_asset(&self, asset_id: &AssetId) -> Result<Option<LiquidAsset>> {
-        lookup_asset(&self, self.asset_db.as_ref(), asset_id)
+        lookup_asset(
+            self,
+            self.asset_db.as_ref().map(AssetRegistryLock::RwLock),
+            asset_id,
+            None,
+        )
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn lookup_registry_asset(&self, asset_id: &AssetId) -> Result<Option<AssetMeta>> {
+        let asset_db = self
+            .asset_db
+            .as_ref()
+            .chain_err(|| "asset registry unavailable")?;
+        Ok(asset_db.read().unwrap().get(asset_id).cloned())
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn list_registry_assets(
+        &self,
+        start_index: usize,
+        limit: usize,
+        sorting: AssetSorting,
+    ) -> Result<(usize, Vec<LiquidAsset>)> {
+        let asset_db = match &self.asset_db {
+            None => return Ok((0, vec![])),
+            Some(db) => db.read().unwrap(),
+        };
+        let (total_num, results) = asset_db.list(start_index, limit, sorting);
+        // Attach on-chain information alongside the registry metadata
+        let results = results
+            .into_iter()
+            .map(|(asset_id, metadata)| {
+                lookup_asset(
+                    self,
+                    Some(AssetRegistryLock::RwLockReadGuard(&asset_db)),
+                    asset_id,
+                    Some(metadata),
+                )?
+                .chain_err(|| "missing registered asset")
+            })
+            .collect::<Result<Vec<_>>>()?;
+        Ok((total_num, results))
+    }
+
+    #[cfg(feature = "liquid")]
+    pub fn search_registry_assets<T, F>(
+        &self,
+        search: &str,
+        limit: usize,
+        mut map: F,
+    ) -> Result<Vec<T>>
+    where
+        F: FnMut(&AssetId, &AssetMeta) -> T,
+    {
+        let asset_db = self
+            .asset_db
+            .as_ref()
+            .chain_err(|| "asset registry unavailable")?;
+        Ok(asset_db
+            .read()
+            .unwrap()
+            .search(search, limit)
+            .into_iter()
+            .map(|(asset_id, metadata)| map(asset_id, metadata))
+            .collect())
     }
 }

src/signal.rs

@@ -1,7 +1,8 @@
 use crossbeam_channel as channel;
 use crossbeam_channel::RecvTimeoutError;
-use std::thread;
-use std::time::Duration;
+use std::time::{Duration, Instant};
+
+use signal_hook::consts::{SIGINT, SIGTERM, SIGUSR1};
 
 use crate::errors::*;
 
@@ -12,9 +13,9 @@ pub struct Waiter {
 
 fn notify(signals: &[i32]) -> channel::Receiver<i32> {
     let (s, r) = channel::bounded(1);
-    let signals =
+    let mut signals =
         signal_hook::iterator::Signals::new(signals).expect("failed to register signal hook");
-    thread::spawn(move || {
+    crate::util::spawn_thread("signal-notifier", move || {
         for signal in signals.forever() {
             s.send(signal)
                 .unwrap_or_else(|_| panic!("failed to send signal {}", signal));
@@ -27,20 +28,26 @@ impl Waiter {
     pub fn start() -> Waiter {
         Waiter {
             receiver: notify(&[
-                signal_hook::SIGINT,
-                signal_hook::SIGTERM,
-                signal_hook::SIGUSR1, // allow external triggering (e.g. via bitcoind `blocknotify`)
+                SIGINT, SIGTERM,
+                SIGUSR1, // allow external triggering (e.g. via bitcoind `blocknotify`)
             ]),
         }
     }
+
     pub fn wait(&self, duration: Duration, accept_sigusr: bool) -> Result<()> {
-        match self.receiver.recv_timeout(duration) {
-            Ok(sig) if sig == signal_hook::SIGUSR1 => {
+        // Determine the deadline time based on the duration, so that it doesn't
+        // get pushed back when wait_deadline() recurses
+        self.wait_deadline(Instant::now() + duration, accept_sigusr)
+    }
+
+    fn wait_deadline(&self, deadline: Instant, accept_sigusr: bool) -> Result<()> {
+        match self.receiver.recv_deadline(deadline) {
+            Ok(sig) if sig == SIGUSR1 => {
                 trace!("notified via SIGUSR1");
                 if accept_sigusr {
                     Ok(())
                 } else {
-                    self.wait(duration, accept_sigusr)
+                    self.wait_deadline(deadline, accept_sigusr)
                 }
             }
             Ok(sig) => bail!(ErrorKind::Interrupt(sig)),

src/util/bincode_tests.rs

@@ -0,0 +1,112 @@
+/*
+
+The tests below show us the following defaults for each method of using bincode.
+
+1. Using bincode::[de]serialize() directly: "function"
+2. Using bincode::config().[de]serialize(): "Config" (deprecated)
+3. Using bincode::options().[de]serialize(): "Options" (currently recommended for v1.3.3)
+
+```
++----------+--------+------------+----------------+------------+
+|          | Endian | Int Length | Allow Trailing | Byte Limit |
++----------+--------+------------+----------------+------------+
+| function | little | fixed      | allow          | unlimited  |
+| Config   | little | fixed      | allow          | unlimited  |
+| Options  | little | variable * | reject *       | unlimited  |
++----------+--------+------------+----------------+------------+
+```
+
+Thus we only need to change the int length from variable to fixed,
+and allow trailing to allow in order to match the previous behavior.
+(note: TxHistory was using Big Endian by explicitly setting it to big.)
+
+*/
+
+use bincode_do_not_use_directly as bincode;
+
+#[test]
+fn bincode_settings() {
+    let value = TestStruct::new();
+    let mut large = [0_u8; 4096];
+    let decoded = [
+        8_u8, 7, 6, 5, 4, 3, 2, 1, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 8, 7, 6, 5, 4, 3, 2, 1, 1,
+        2, 3, 4, 5, 6, 7, 8, 12, 0, 0, 0, 0, 0, 0, 0, 72, 101, 108, 108, 111, 32, 87, 111, 114,
+        108, 100, 33,
+    ];
+    large[0..56].copy_from_slice(&decoded);
+
+    // Using functions: Little endian, Fixint, Allow trailing, Unlimited
+    assert_eq!(bincode::serialize(&value).unwrap(), &decoded);
+    assert_eq!(bincode::deserialize::<TestStruct>(&large).unwrap(), value);
+
+    // Using Config (deprecated)
+    // Little endian, fixint, Allow trailing, Unlimited
+    #[allow(deprecated)]
+    {
+        assert_eq!(bincode::config().serialize(&value).unwrap(), &decoded);
+        assert_eq!(
+            bincode::config().deserialize::<TestStruct>(&large).unwrap(),
+            value
+        );
+    }
+
+    // Using Options
+    // Little endian, VARINT (different), Reject trailing (different), unlimited
+    use bincode::Options;
+    assert_eq!(
+        bincode::options()
+            .with_fixint_encoding()
+            .allow_trailing_bytes()
+            .serialize(&value)
+            .unwrap(),
+        &decoded
+    );
+    assert_eq!(
+        bincode::options()
+            .with_fixint_encoding()
+            .allow_trailing_bytes()
+            .deserialize::<TestStruct>(&large)
+            .unwrap(),
+        value
+    );
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+struct TestStruct {
+    a: u64,
+    b: [u8; 8],
+    c: TestData,
+    d: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+enum TestData {
+    Foo(FooStruct),
+    Bar(BarStruct),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+struct FooStruct {
+    a: u64,
+    b: [u8; 8],
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+struct BarStruct {
+    a: u64,
+    b: [u8; 8],
+}
+
+impl TestStruct {
+    fn new() -> Self {
+        Self {
+            a: 0x0102030405060708,
+            b: [1, 2, 3, 4, 5, 6, 7, 8],
+            c: TestData::Foo(FooStruct {
+                a: 0x0102030405060708,
+                b: [1, 2, 3, 4, 5, 6, 7, 8],
+            }),
+            d: String::from("Hello World!"),
+        }
+    }
+}

src/util/bincode_util.rs

@@ -0,0 +1,70 @@
+//! This module creates two sets of serialize and deserialize for bincode.
+//! They explicitly spell out the bincode settings so that switching to
+//! new versions in the future is less error prone.
+//!
+//! This is a list of all the row types and their settings for bincode.
+//! +--------------+--------+------------+----------------+------------+
+//! |              | Endian | Int Length | Allow Trailing | Byte Limit |
+//! +--------------+--------+------------+----------------+------------+
+//! | TxHistoryRow | big    | fixed      | allow          | unlimited  |
+//! | All others   | little | fixed      | allow          | unlimited  |
+//! +--------------+--------+------------+----------------+------------+
+
+// We only want people to use bincode_util
+use bincode::Options;
+use bincode_do_not_use_directly as bincode;
+
+pub fn serialize_big<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
+where
+    T: ?Sized + serde::Serialize,
+{
+    big_endian().serialize(value)
+}
+
+pub fn deserialize_big<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
+where
+    T: serde::Deserialize<'a>,
+{
+    big_endian().deserialize(bytes)
+}
+
+pub fn serialize_little<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
+where
+    T: ?Sized + serde::Serialize,
+{
+    little_endian().serialize(value)
+}
+
+pub fn deserialize_little<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
+where
+    T: serde::Deserialize<'a>,
+{
+    little_endian().deserialize(bytes)
+}
+
+/// This is the default settings for Options,
+/// but all explicitly spelled out, except for endianness.
+/// The following functions will add endianness.
+#[inline]
+fn options() -> impl Options {
+    bincode::options()
+        .with_fixint_encoding()
+        .with_no_limit()
+        .allow_trailing_bytes()
+}
+
+/// Adding the endian flag for big endian
+#[inline]
+fn big_endian() -> impl Options {
+    options().with_big_endian()
+}
+
+/// Adding the endian flag for little endian
+#[inline]
+fn little_endian() -> impl Options {
+    options().with_little_endian()
+}
+
+#[cfg(test)]
+#[path = "./bincode_tests.rs"]
+mod bincode_tests;

src/util/block.rs

@@ -1,15 +1,18 @@
-use crate::chain::BlockHeader;
+use bitcoin::hashes::Hash;
+
+use crate::chain::{BlockHash, BlockHeader};
 use crate::errors::*;
 use crate::new_index::BlockEntry;
 
-use bitcoin::{BitcoinHash, BlockHash};
-
 use std::collections::HashMap;
 use std::fmt;
 use std::iter::FromIterator;
 use std::slice;
+use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime as DateTime;
 
+const MTP_SPAN: usize = 11;
+
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct BlockId {
     pub height: usize,
@@ -50,13 +53,13 @@ impl HeaderEntry {
 
 impl fmt::Debug for HeaderEntry {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let last_block_time = DateTime::from_unix_timestamp(self.header().time as i64);
+        let last_block_time = DateTime::from_unix_timestamp(self.header().time as i64).unwrap();
         write!(
             f,
             "hash={} height={} @ {}",
             self.hash(),
             self.height(),
-            last_block_time.format(time::Format::Rfc3339),
+            last_block_time.format(&Rfc3339).unwrap(),
         )
     }
 }
@@ -72,7 +75,7 @@ impl HeaderList {
         HeaderList {
             headers: vec![],
             heights: HashMap::new(),
-            tip: BlockHash::default(),
+            tip: BlockHash::all_zeros(),
         }
     }
 
@@ -88,14 +91,14 @@ impl HeaderList {
 
         let mut blockhash = tip_hash;
         let mut headers_chain: Vec<BlockHeader> = vec![];
-        let null_hash = BlockHash::default();
+        let null_hash = BlockHash::all_zeros();
 
         while blockhash != null_hash {
             let header = headers_map.remove(&blockhash).unwrap_or_else(|| {
                 panic!(
                     "missing expected blockhash in headers map: {:?}, pointed from: {:?}",
                     blockhash,
-                    headers_chain.last().map(|h| h.bitcoin_hash())
+                    headers_chain.last().map(|h| h.block_hash())
                 )
             });
             blockhash = header.prev_blockhash;
@@ -122,7 +125,7 @@ impl HeaderList {
         }
         let hashed_headers =
             Vec::<HashedHeader>::from_iter(new_headers.into_iter().map(|header| HashedHeader {
-                blockhash: header.bitcoin_hash(),
+                blockhash: header.block_hash(),
                 header,
             }));
         for i in 1..hashed_headers.len() {
@@ -135,7 +138,7 @@ impl HeaderList {
             Some(h) => h.header.prev_blockhash,
             None => return vec![], // hashed_headers is empty
         };
-        let null_hash = BlockHash::default();
+        let null_hash = BlockHash::all_zeros();
         let new_height: usize = if prev_blockhash == null_hash {
             0
         } else {
@@ -145,7 +148,7 @@ impl HeaderList {
                 + 1
         };
         (new_height..)
-            .zip(hashed_headers.into_iter())
+            .zip(hashed_headers)
             .map(|(height, hashed_header)| HeaderEntry {
                 height,
                 hash: hashed_header.blockhash,
@@ -154,7 +157,12 @@ impl HeaderList {
             .collect()
     }
 
-    pub fn apply(&mut self, new_headers: Vec<HeaderEntry>) {
+    /// Returns any rolled back blocks in order from old tip first and first block in the fork is last
+    /// It also returns the blockhash of the post-rollback tip.
+    pub fn apply(
+        &mut self,
+        new_headers: Vec<HeaderEntry>,
+    ) -> (Vec<HeaderEntry>, Option<BlockHash>) {
         // new_headers[i] -> new_headers[i - 1] (i.e. new_headers.last() is the tip)
         for i in 1..new_headers.len() {
             assert_eq!(new_headers[i - 1].height() + 1, new_headers[i].height());
@@ -169,19 +177,27 @@ impl HeaderList {
                 let expected_prev_blockhash = if height > 0 {
                     *self.headers[height - 1].hash()
                 } else {
-                    BlockHash::default()
+                    BlockHash::all_zeros()
                 };
                 assert_eq!(entry.header().prev_blockhash, expected_prev_blockhash);
                 height
             }
-            None => return,
+            None => return (vec![], None),
         };
         debug!(
             "applying {} new headers from height {}",
             new_headers.len(),
             new_height
         );
-        let _removed = self.headers.split_off(new_height); // keep [0..new_height) entries
+        let mut removed = self.headers.split_off(new_height); // keep [0..new_height) entries
+
+        // If we reorged, we should return the last blockhash before adding the new chain's blockheaders.
+        let reorged_tip = if !removed.is_empty() {
+            self.headers.last().map(|be| be.hash()).cloned()
+        } else {
+            None
+        };
+
         for new_header in new_headers {
             let height = new_header.height();
             assert_eq!(height, self.headers.len());
@@ -189,6 +205,8 @@ impl HeaderList {
             self.headers.push(new_header);
             self.heights.insert(self.tip, height);
         }
+        removed.reverse();
+        (removed, reorged_tip)
     }
 
     pub fn header_by_blockhash(&self, blockhash: &BlockHash) -> Option<&HeaderEntry> {
@@ -202,9 +220,8 @@ impl HeaderList {
     }
 
     pub fn header_by_height(&self, height: usize) -> Option<&HeaderEntry> {
-        self.headers.get(height).map(|entry| {
+        self.headers.get(height).inspect(|entry| {
             assert_eq!(entry.height(), height);
-            entry
         })
     }
 
@@ -215,7 +232,10 @@ impl HeaderList {
     pub fn tip(&self) -> &BlockHash {
         assert_eq!(
             self.tip,
-            self.headers.last().map(|h| *h.hash()).unwrap_or_default()
+            self.headers
+                .last()
+                .map(|h| *h.hash())
+                .unwrap_or(BlockHash::all_zeros())
         );
         &self.tip
     }
@@ -228,9 +248,26 @@ impl HeaderList {
         self.headers.is_empty()
     }
 
-    pub fn iter(&self) -> slice::Iter<HeaderEntry> {
+    pub fn iter(&self) -> slice::Iter<'_, HeaderEntry> {
         self.headers.iter()
     }
+
+    /// Get the Median Time Past
+    pub fn get_mtp(&self, height: usize) -> u32 {
+        // Use the timestamp as the mtp of the genesis block.
+        // Matches bitcoind's behaviour: bitcoin-cli getblock `bitcoin-cli getblockhash 0` | jq '.time == .mediantime'
+        if height == 0 {
+            self.headers.first().unwrap().header.time
+        } else if height > self.len() - 1 {
+            0
+        } else {
+            let mut timestamps = (height.saturating_sub(MTP_SPAN - 1)..=height)
+                .map(|p_height| self.headers.get(p_height).unwrap().header.time)
+                .collect::<Vec<_>>();
+            timestamps.sort_unstable();
+            timestamps[timestamps.len() / 2]
+        }
+    }
 }
 
 #[derive(Serialize, Deserialize)]
@@ -269,13 +306,18 @@ pub struct BlockMeta {
 pub struct BlockHeaderMeta {
     pub header_entry: HeaderEntry,
     pub meta: BlockMeta,
+    pub mtp: u32,
 }
 
 impl From<&BlockEntry> for BlockMeta {
     fn from(b: &BlockEntry) -> BlockMeta {
+        #[cfg(not(feature = "liquid"))]
+        let weight = b.block.weight().to_wu() as u32;
+        #[cfg(feature = "liquid")]
+        let weight = b.block.weight() as u32;
         BlockMeta {
             tx_count: b.block.txdata.len() as u32,
-            weight: b.block.get_weight() as u32,
+            weight,
             size: b.size,
         }
     }

src/util/electrum_merkle.rs

@@ -1,8 +1,7 @@
-use bitcoin::hashes::{sha256d::Hash as Sha256dHash, Hash};
-use bitcoin::{BlockHash, Txid};
-
+use crate::chain::{BlockHash, Txid};
 use crate::errors::*;
 use crate::new_index::ChainQuery;
+use bitcoin::hashes::{sha256d::Hash as Sha256dHash, Hash};
 
 pub fn get_tx_merkle_proof(
     chain: &ChainQuery,
@@ -10,7 +9,7 @@ pub fn get_tx_merkle_proof(
     block_hash: &BlockHash,
 ) -> Result<(Vec<Sha256dHash>, usize)> {
     let txids = chain
-        .get_block_txids(&block_hash)
+        .get_block_txids(block_hash)
         .chain_err(|| format!("missing block txids for #{}", block_hash))?;
     let pos = txids
         .iter()

src/util/fees.rs

@@ -12,7 +12,10 @@ pub struct TxFeeInfo {
 impl TxFeeInfo {
     pub fn new(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>, network: Network) -> Self {
         let fee = get_tx_fee(tx, prevouts, network);
-        let vsize = tx.get_weight() / 4;
+        #[cfg(not(feature = "liquid"))]
+        let vsize = tx.weight().to_wu() / 4;
+        #[cfg(feature = "liquid")]
+        let vsize = tx.weight() / 4;
 
         TxFeeInfo {
             fee,
@@ -24,12 +27,15 @@ impl TxFeeInfo {
 
 #[cfg(not(feature = "liquid"))]
 pub fn get_tx_fee(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>, _network: Network) -> u64 {
-    if tx.is_coin_base() {
+    if tx.is_coinbase() {
         return 0;
     }
 
-    let total_in: u64 = prevouts.values().map(|prevout| prevout.value).sum();
-    let total_out: u64 = tx.output.iter().map(|vout| vout.value).sum();
+    let total_in: u64 = prevouts
+        .values()
+        .map(|prevout| prevout.value.to_sat())
+        .sum();
+    let total_out: u64 = tx.output.iter().map(|vout| vout.value.to_sat()).sum();
     total_in - total_out
 }
 
@@ -43,18 +49,18 @@ pub fn make_fee_histogram(mut entries: Vec<&TxFeeInfo>) -> Vec<(f32, u32)> {
 
     let mut histogram = vec![];
     let mut bin_size = 0;
-    let mut last_fee_rate = None;
+    let mut last_fee_rate = 0.0;
     for e in entries.iter().rev() {
-        bin_size += e.vsize;
-        if bin_size > VSIZE_BIN_WIDTH && last_fee_rate.map_or(true, |last| e.fee_per_vbyte < last) {
-            // vsize of transactions paying >= e.fee_per_vbyte
-            histogram.push((e.fee_per_vbyte, bin_size));
+        if bin_size > VSIZE_BIN_WIDTH && last_fee_rate != e.fee_per_vbyte {
+            // vsize of transactions paying >= last_fee_rate
+            histogram.push((last_fee_rate, bin_size));
             bin_size = 0;
         }
-        last_fee_rate = Some(e.fee_per_vbyte);
+        last_fee_rate = e.fee_per_vbyte;
+        bin_size += e.vsize;
     }
-    if let Some(fee_rate) = last_fee_rate {
-        histogram.push((fee_rate, bin_size));
+    if bin_size > 0 {
+        histogram.push((last_fee_rate, bin_size));
     }
     histogram
 }

src/util/mod.rs

@@ -2,19 +2,25 @@ mod block;
 mod script;
 mod transaction;
 
+pub mod bincode_util;
 pub mod electrum_merkle;
 pub mod fees;
 
 pub use self::block::{BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, HeaderEntry, HeaderList};
 pub use self::fees::get_tx_fee;
-pub use self::script::{get_innerscripts, get_script_asm, script_to_address};
+pub use self::script::{
+    get_innerscripts, IsProvablyUnspendable, ScriptToAddr, ScriptToAsm, SegwitDetection,
+};
 pub use self::transaction::{
-    extract_tx_prevouts, has_prevout, is_coinbase, is_spendable, TransactionStatus, TxInput,
+    extract_tx_prevouts, has_prevout, is_coinbase, is_spendable, serialize_outpoint,
+    sigops::transaction_sigop_count, TransactionStatus, TxInput,
 };
 
 use std::collections::HashMap;
-use std::sync::mpsc::{channel, sync_channel, Receiver, Sender, SyncSender};
-use std::thread;
+use std::sync::atomic::AtomicUsize;
+use std::sync::mpsc::{channel, Receiver, Sender};
+use std::sync::Mutex;
+use std::thread::{self, ThreadId};
 
 use crate::chain::BlockHeader;
 use bitcoin::hashes::sha256d::Hash as Sha256dHash;
@@ -34,26 +40,38 @@ pub fn full_hash(hash: &[u8]) -> FullHash {
 }
 
 pub struct SyncChannel<T> {
-    tx: SyncSender<T>,
-    rx: Receiver<T>,
+    tx: Option<crossbeam_channel::Sender<T>>,
+    rx: Option<crossbeam_channel::Receiver<T>>,
 }
 
 impl<T> SyncChannel<T> {
     pub fn new(size: usize) -> SyncChannel<T> {
-        let (tx, rx) = sync_channel(size);
-        SyncChannel { tx, rx }
+        let (tx, rx) = crossbeam_channel::bounded(size);
+        SyncChannel {
+            tx: Some(tx),
+            rx: Some(rx),
+        }
     }
 
-    pub fn sender(&self) -> SyncSender<T> {
-        self.tx.clone()
+    pub fn sender(&self) -> crossbeam_channel::Sender<T> {
+        self.tx.as_ref().expect("No Sender").clone()
     }
 
-    pub fn receiver(&self) -> &Receiver<T> {
-        &self.rx
+    pub fn receiver(&self) -> &crossbeam_channel::Receiver<T> {
+        self.rx.as_ref().expect("No Receiver")
     }
 
-    pub fn into_receiver(self) -> Receiver<T> {
-        self.rx
+    pub fn into_receiver(self) -> crossbeam_channel::Receiver<T> {
+        self.rx.expect("No Receiver")
+    }
+
+    /// This drops the sender and receiver, causing all other methods to panic.
+    ///
+    /// Use only when you know that the channel will no longer be used.
+    /// ie. shutdown.
+    pub fn close(&mut self) -> Option<crossbeam_channel::Receiver<T>> {
+        self.tx.take();
+        self.rx.take()
     }
 }
 
@@ -81,15 +99,58 @@ impl<T> Channel<T> {
     }
 }
 
-pub fn spawn_thread<F, T>(name: &str, f: F) -> thread::JoinHandle<T>
+/// This static HashMap contains all the threads spawned with [`spawn_thread`] with their name
+#[inline]
+pub fn with_spawned_threads<F>(f: F)
+where
+    F: FnOnce(&mut HashMap<ThreadId, String>),
+{
+    lazy_static! {
+        static ref SPAWNED_THREADS: Mutex<HashMap<ThreadId, String>> = Mutex::new(HashMap::new());
+    }
+    let mut lock = match SPAWNED_THREADS.lock() {
+        Ok(threads) => threads,
+        // There's no possible broken state
+        Err(threads) => {
+            warn!("SPAWNED_THREADS is in a poisoned state! Be wary of incorrect logs!");
+            threads.into_inner()
+        }
+    };
+    f(&mut lock)
+}
+
+pub fn spawn_thread<F, T>(prefix: &str, do_work: F) -> thread::JoinHandle<T>
 where
     F: FnOnce() -> T,
     F: Send + 'static,
     T: Send + 'static,
 {
+    static THREAD_COUNTER: AtomicUsize = AtomicUsize::new(0);
+    let counter = THREAD_COUNTER.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
     thread::Builder::new()
-        .name(name.to_owned())
-        .spawn(f)
+        .name(format!("{}-{}", prefix, counter))
+        .spawn(move || {
+            let thread = std::thread::current();
+            let name = thread.name().unwrap();
+            let id = thread.id();
+
+            trace!("[THREAD] GETHASHMAP INSERT | {name} {id:?}");
+            with_spawned_threads(|threads| {
+                threads.insert(id, name.to_owned());
+            });
+            trace!("[THREAD] START WORK        | {name} {id:?}");
+
+            let result = do_work();
+
+            trace!("[THREAD] FINISHED WORK     | {name} {id:?}");
+            trace!("[THREAD] GETHASHMAP REMOVE | {name} {id:?}");
+            with_spawned_threads(|threads| {
+                threads.remove(&id);
+            });
+            trace!("[THREAD] HASHMAP REMOVED   | {name} {id:?}");
+
+            result
+        })
         .unwrap()
 }
 
@@ -113,18 +174,18 @@ impl BoolThen for bool {
 
 pub fn create_socket(addr: &SocketAddr) -> Socket {
     let domain = match &addr {
-        SocketAddr::V4(_) => Domain::ipv4(),
-        SocketAddr::V6(_) => Domain::ipv6(),
+        SocketAddr::V4(_) => Domain::IPV4,
+        SocketAddr::V6(_) => Domain::IPV6,
     };
     let socket =
-        Socket::new(domain, Type::stream(), Some(Protocol::tcp())).expect("creating socket failed");
+        Socket::new(domain, Type::STREAM, Some(Protocol::TCP)).expect("creating socket failed");
 
     #[cfg(unix)]
     socket
         .set_reuse_port(true)
         .expect("cannot enable SO_REUSEPORT");
 
-    socket.bind(&addr.clone().into()).expect("cannot bind");
+    socket.bind(&(*addr).into()).expect("cannot bind");
 
     socket
 }
@@ -135,28 +196,28 @@ pub fn create_socket(addr: &SocketAddr) -> Socket {
 ///
 /// Copied from https://github.com/rust-bitcoin/rust-bitcoincore-rpc/blob/master/json/src/lib.rs
 pub mod serde_hex {
-    use bitcoin::hashes::hex::{FromHex, ToHex};
+    use bitcoin::hashes::hex::FromHex;
     use serde::de::Error;
     use serde::{Deserializer, Serializer};
 
-    pub fn serialize<S: Serializer>(b: &Vec<u8>, s: S) -> Result<S::Ok, S::Error> {
-        s.serialize_str(&b.to_hex())
+    pub fn serialize<S: Serializer>(b: &[u8], s: S) -> Result<S::Ok, S::Error> {
+        s.serialize_str(&hex::encode(b))
     }
 
     pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
         let hex_str: String = ::serde::Deserialize::deserialize(d)?;
-        Ok(FromHex::from_hex(&hex_str).map_err(D::Error::custom)?)
+        FromHex::from_hex(&hex_str).map_err(D::Error::custom)
     }
 
     pub mod opt {
-        use bitcoin::hashes::hex::{FromHex, ToHex};
+        use bitcoin::hashes::hex::FromHex;
         use serde::de::Error;
         use serde::{Deserializer, Serializer};
 
         pub fn serialize<S: Serializer>(b: &Option<Vec<u8>>, s: S) -> Result<S::Ok, S::Error> {
             match *b {
                 None => s.serialize_none(),
-                Some(ref b) => s.serialize_str(&b.to_hex()),
+                Some(ref b) => s.serialize_str(&hex::encode(b)),
             }
         }
 

src/util/script.rs

@@ -1,38 +1,135 @@
-use bitcoin::blockdata::script::{Instruction::PushBytes, Script};
-
 #[cfg(feature = "liquid")]
 use elements::address as elements_address;
 
-use crate::chain::Network;
-use crate::chain::{TxIn, TxOut};
+use crate::chain::{script, Network, Script, TxIn, TxOut};
+use script::Instruction::PushBytes;
 
 pub struct InnerScripts {
     pub redeem_script: Option<Script>,
     pub witness_script: Option<Script>,
 }
 
-pub fn script_to_address(script: &Script, network: Network) -> Option<String> {
-    match network {
-        #[cfg(feature = "liquid")]
-        Network::Liquid | Network::LiquidRegtest => {
-            elements_address::Address::from_script(script, None, network.address_params())
-                .map(|a| a.to_string())
+pub trait IsProvablyUnspendable {
+    fn is_provably_unspendable_(&self) -> bool;
+}
+
+#[cfg(not(feature = "liquid"))]
+impl IsProvablyUnspendable for bitcoin::Script {
+    // is_provably_unspendable() is deprecated in rust-bitcoin
+    // so we re-implement it here. Copy pasted.
+    fn is_provably_unspendable_(&self) -> bool {
+        use bitcoin::blockdata::opcodes::{
+            Class::{IllegalOp, ReturnOp},
+            ClassifyContext, Opcode,
+        };
+
+        match self.as_bytes().first() {
+            Some(b) => {
+                let first = Opcode::from(*b);
+                let class = first.classify(ClassifyContext::Legacy);
+
+                class == ReturnOp || class == IllegalOp
+            }
+            None => false,
         }
-        _ => bitcoin::Address::from_script(script, network.into()).map(|s| s.to_string()),
     }
 }
 
-pub fn get_script_asm(script: &Script) -> String {
-    let asm = format!("{:?}", script);
-    (&asm[7..asm.len() - 1]).to_string()
+#[cfg(feature = "liquid")]
+impl IsProvablyUnspendable for elements::Script {
+    #[inline(always)]
+    fn is_provably_unspendable_(&self) -> bool {
+        // Not deprecated yet
+        self.is_provably_unspendable()
+    }
+}
+
+// Extension trait for segwit script detection that works across bitcoin and elements
+pub trait SegwitDetection {
+    fn segwit_is_p2wpkh(&self) -> bool;
+    fn segwit_is_p2wsh(&self) -> bool;
+    fn segwit_is_p2tr(&self) -> bool;
+}
+
+#[cfg(not(feature = "liquid"))]
+impl SegwitDetection for bitcoin::Script {
+    fn segwit_is_p2wpkh(&self) -> bool {
+        self.is_p2wpkh()
+    }
+    fn segwit_is_p2wsh(&self) -> bool {
+        self.is_p2wsh()
+    }
+    fn segwit_is_p2tr(&self) -> bool {
+        self.is_p2tr()
+    }
+}
+
+#[cfg(not(feature = "liquid"))]
+impl SegwitDetection for bitcoin::ScriptBuf {
+    fn segwit_is_p2wpkh(&self) -> bool {
+        self.is_p2wpkh()
+    }
+    fn segwit_is_p2wsh(&self) -> bool {
+        self.is_p2wsh()
+    }
+    fn segwit_is_p2tr(&self) -> bool {
+        self.is_p2tr()
+    }
+}
+
+#[cfg(feature = "liquid")]
+impl SegwitDetection for elements::Script {
+    fn segwit_is_p2wpkh(&self) -> bool {
+        self.is_v0_p2wpkh()
+    }
+    fn segwit_is_p2wsh(&self) -> bool {
+        self.is_v0_p2wsh()
+    }
+    fn segwit_is_p2tr(&self) -> bool {
+        self.is_v1_p2tr()
+    }
+}
+
+pub trait ScriptToAsm: std::fmt::Debug {
+    fn to_asm(&self) -> String {
+        let asm = format!("{:?}", self);
+        asm[7..asm.len() - 1].to_string()
+    }
+}
+impl ScriptToAsm for bitcoin::Script {}
+impl ScriptToAsm for bitcoin::ScriptBuf {}
+#[cfg(feature = "liquid")]
+impl ScriptToAsm for elements::Script {}
+
+pub trait ScriptToAddr {
+    fn to_address_str(&self, network: Network) -> Option<String>;
+}
+#[cfg(not(feature = "liquid"))]
+impl ScriptToAddr for bitcoin::Script {
+    fn to_address_str(&self, network: Network) -> Option<String> {
+        bitcoin::Address::from_script(self, bitcoin::Network::from(network))
+            .ok()
+            .map(|s| s.to_string())
+    }
+}
+#[cfg(feature = "liquid")]
+impl ScriptToAddr for elements::Script {
+    fn to_address_str(&self, network: Network) -> Option<String> {
+        elements_address::Address::from_script(self, None, network.address_params())
+            .map(|a| a.to_string())
+    }
 }
 
 // Returns the witnessScript in the case of p2wsh, or the redeemScript in the case of p2sh.
 pub fn get_innerscripts(txin: &TxIn, prevout: &TxOut) -> InnerScripts {
     // Wrapped redeemScript for P2SH spends
     let redeem_script = if prevout.script_pubkey.is_p2sh() {
-        if let Some(PushBytes(redeemscript)) = txin.script_sig.iter(true).last() {
-            Some(Script::from(redeemscript.to_vec()))
+        if let Some(Ok(PushBytes(redeemscript))) = txin.script_sig.instructions().last() {
+            #[cfg(not(feature = "liquid"))]
+            let bytes = redeemscript.as_bytes().to_vec();
+            #[cfg(feature = "liquid")]
+            let bytes = redeemscript.to_vec();
+            Some(Script::from(bytes))
         } else {
             None
         }
@@ -41,13 +138,53 @@ pub fn get_innerscripts(txin: &TxIn, prevout: &TxOut) -> InnerScripts {
     };
 
     // Wrapped witnessScript for P2WSH or P2SH-P2WSH spends
-    let witness_script = if prevout.script_pubkey.is_v0_p2wsh()
-        || redeem_script.as_ref().map_or(false, |s| s.is_v0_p2wsh())
+    let witness_script = if prevout.script_pubkey.segwit_is_p2wsh()
+        || prevout.script_pubkey.segwit_is_p2tr()
+        || redeem_script.as_ref().is_some_and(|s| s.segwit_is_p2wsh())
     {
         let witness = &txin.witness;
         #[cfg(feature = "liquid")]
         let witness = &witness.script_witness;
-        witness.iter().last().cloned().map(Script::from)
+
+        // rust-bitcoin returns witness items as a [u8] slice, while rust-elements returns a Vec<u8>
+        #[cfg(not(feature = "liquid"))]
+        let wit_to_vec = Vec::from;
+        #[cfg(feature = "liquid")]
+        let wit_to_vec = Clone::clone;
+
+        let inner_script_slice = if prevout.script_pubkey.segwit_is_p2tr() {
+            // Witness stack is potentially very large
+            // so we avoid to_vec() or iter().collect() for performance
+            let w_len = witness.len();
+            witness
+                .last()
+                // Get the position of the script spend script (if it exists)
+                .map(|last_elem| {
+                    // From BIP341:
+                    // If there are at least two witness elements, and the first byte of
+                    // the last element is 0x50, this last element is called annex a
+                    // and is removed from the witness stack.
+                    if w_len >= 2 && last_elem.first().filter(|&&v| v == 0x50).is_some() {
+                        // account for the extra item removed from the end
+                        3
+                    } else {
+                        // otherwise script is 2nd from last
+                        2
+                    }
+                })
+                // Convert to None if not script spend
+                // Note: Option doesn't have filter_map() method
+                .filter(|&script_pos_from_last| w_len >= script_pos_from_last)
+                .and_then(|script_pos_from_last| {
+                    // Can't use second_to_last() since it might be 3rd to last
+                    #[allow(clippy::iter_nth)]
+                    witness.iter().nth(w_len - script_pos_from_last)
+                })
+        } else {
+            witness.last()
+        };
+
+        inner_script_slice.map(wit_to_vec).map(Script::from)
     } else {
         None
     };

src/util/transaction.rs

@@ -1,17 +1,19 @@
-use bitcoin::{BlockHash, Txid};
-
-#[cfg(feature = "liquid")]
-use bitcoin::hashes::hex::FromHex;
-
-use crate::chain::{OutPoint, Transaction, TxIn, TxOut};
-use crate::util::BlockId;
+use crate::chain::{BlockHash, OutPoint, Transaction, TxIn, TxOut, Txid};
+use crate::errors;
+use crate::util::{BlockId, IsProvablyUnspendable};
 
 use std::collections::HashMap;
 
 #[cfg(feature = "liquid")]
 lazy_static! {
     static ref REGTEST_INITIAL_ISSUANCE_PREVOUT: Txid =
-        Txid::from_hex("50cdc410c9d0d61eeacc531f52d2c70af741da33af127c364e52ac1ee7c030a5").unwrap();
+        "50cdc410c9d0d61eeacc531f52d2c70af741da33af127c364e52ac1ee7c030a5"
+            .parse()
+            .unwrap();
+    static ref TESTNET_INITIAL_ISSUANCE_PREVOUT: Txid =
+        "0c52d2526a5c9f00e9fb74afd15dd3caaf17c823159a514f929ae25193a43a52"
+            .parse()
+            .unwrap();
 }
 
 #[derive(Serialize, Deserialize)]
@@ -30,7 +32,7 @@ impl From<Option<BlockId>> for TransactionStatus {
         match blockid {
             Some(b) => TransactionStatus {
                 confirmed: true,
-                block_height: Some(b.height as usize),
+                block_height: Some(b.height),
                 block_hash: Some(b.hash),
                 block_time: Some(b.time),
             },
@@ -47,7 +49,7 @@ impl From<Option<BlockId>> for TransactionStatus {
 #[derive(Serialize, Deserialize)]
 pub struct TxInput {
     pub txid: Txid,
-    pub vin: u16,
+    pub vin: u32,
 }
 
 pub fn is_coinbase(txin: &TxIn) -> bool {
@@ -63,33 +65,328 @@ pub fn has_prevout(txin: &TxIn) -> bool {
     #[cfg(feature = "liquid")]
     return !txin.is_coinbase()
         && !txin.is_pegin
-        && txin.previous_output.txid != *REGTEST_INITIAL_ISSUANCE_PREVOUT;
+        && txin.previous_output.txid != *REGTEST_INITIAL_ISSUANCE_PREVOUT
+        && txin.previous_output.txid != *TESTNET_INITIAL_ISSUANCE_PREVOUT;
 }
 
 pub fn is_spendable(txout: &TxOut) -> bool {
     #[cfg(not(feature = "liquid"))]
-    return !txout.script_pubkey.is_provably_unspendable();
+    return !txout.script_pubkey.is_provably_unspendable_();
     #[cfg(feature = "liquid")]
-    return !txout.is_fee() && !txout.script_pubkey.is_provably_unspendable();
+    return !txout.is_fee() && !txout.script_pubkey.is_provably_unspendable_();
 }
 
+/// Extract the previous TxOuts of a Transaction's TxIns
+///
+/// # Errors
+///
+/// This function MUST NOT return an error variant when allow_missing is true.
+/// If allow_missing is false, it will return an error when any Outpoint is
+/// missing from the keys of the txos argument's HashMap.
 pub fn extract_tx_prevouts<'a>(
     tx: &Transaction,
     txos: &'a HashMap<OutPoint, TxOut>,
-    allow_missing: bool,
-) -> HashMap<u32, &'a TxOut> {
+) -> Result<HashMap<u32, &'a TxOut>, errors::Error> {
     tx.input
         .iter()
         .enumerate()
         .filter(|(_, txi)| has_prevout(txi))
-        .filter_map(|(index, txi)| {
-            Some((
+        .map(|(index, txi)| {
+            Ok((
                 index as u32,
-                txos.get(&txi.previous_output).or_else(|| {
-                    assert!(allow_missing, "missing outpoint {:?}", txi.previous_output);
-                    None
-                })?,
+                match txos.get(&txi.previous_output) {
+                    Some(txo) => txo,
+                    None => {
+                        return Err(format!("missing outpoint {:?}", txi.previous_output).into());
+                    }
+                },
             ))
         })
         .collect()
 }
+
+pub fn serialize_outpoint<S>(outpoint: &OutPoint, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::ser::Serializer,
+{
+    use serde::ser::SerializeStruct;
+    let mut s = serializer.serialize_struct("OutPoint", 2)?;
+    s.serialize_field("txid", &outpoint.txid)?;
+    s.serialize_field("vout", &outpoint.vout)?;
+    s.end()
+}
+
+pub(super) mod sigops {
+    use crate::chain::{
+        opcodes::all::{OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY},
+        script::{self, Instruction},
+        Transaction, TxOut, Witness,
+    };
+    #[cfg(not(feature = "liquid"))]
+    use bitcoin::opcodes::Opcode;
+    #[cfg(feature = "liquid")]
+    use elements::opcodes::All as Opcode;
+    use std::collections::HashMap;
+
+    /// Get sigop count for transaction. prevout_map must have all the prevouts.
+    pub fn transaction_sigop_count(
+        tx: &Transaction,
+        prevout_map: &HashMap<u32, &TxOut>,
+    ) -> Result<usize, script::Error> {
+        let input_count = tx.input.len();
+        let mut prevouts = Vec::with_capacity(input_count);
+
+        #[cfg(not(feature = "liquid"))]
+        let is_coinbase_or_pegin = tx.is_coinbase();
+        #[cfg(feature = "liquid")]
+        let is_coinbase_or_pegin = tx.is_coinbase() || tx.input.iter().any(|input| input.is_pegin);
+
+        if !is_coinbase_or_pegin {
+            for idx in 0..input_count {
+                prevouts.push(
+                    *prevout_map
+                        .get(&(idx as u32))
+                        .ok_or(script::Error::EarlyEndOfScript)?,
+                );
+            }
+        }
+
+        // coinbase tx won't use prevouts so it can be empty.
+        get_sigop_cost(tx, &prevouts, true, true)
+    }
+
+    fn decode_pushnum(op: &Opcode) -> Option<u8> {
+        // 81 = OP_1, 96 = OP_16
+        // 81 -> 1, so... 81 - 80 -> 1
+        #[cfg(not(feature = "liquid"))]
+        let self_u8 = op.to_u8();
+        #[cfg(feature = "liquid")]
+        let self_u8 = op.into_u8();
+        match self_u8 {
+            81..=96 => Some(self_u8 - 80),
+            _ => None,
+        }
+    }
+
+    fn count_sigops(script: &script::Script, accurate: bool) -> usize {
+        let mut n = 0;
+        let mut pushnum_cache = None;
+        for inst in script.instructions() {
+            match inst {
+                Ok(Instruction::Op(opcode)) => {
+                    match opcode {
+                        OP_CHECKSIG | OP_CHECKSIGVERIFY => {
+                            n += 1;
+                        }
+                        OP_CHECKMULTISIG | OP_CHECKMULTISIGVERIFY => {
+                            match (accurate, pushnum_cache) {
+                                (true, Some(pushnum)) => {
+                                    // Add the number of pubkeys in the multisig as sigop count
+                                    n += usize::from(pushnum);
+                                }
+                                _ => {
+                                    // MAX_PUBKEYS_PER_MULTISIG from Bitcoin Core
+                                    // https://github.com/bitcoin/bitcoin/blob/v25.0/src/script/script.h#L29-L30
+                                    n += 20;
+                                }
+                            }
+                        }
+                        _ => {
+                            pushnum_cache = decode_pushnum(&opcode);
+                        }
+                    }
+                }
+                // We ignore errors as well as pushdatas
+                _ => {
+                    pushnum_cache = None;
+                }
+            }
+        }
+
+        n
+    }
+
+    /// Get the sigop count for legacy transactions
+    fn get_legacy_sigop_count(tx: &Transaction) -> usize {
+        let mut n = 0;
+        for input in &tx.input {
+            n += count_sigops(&input.script_sig, false);
+        }
+        for output in &tx.output {
+            n += count_sigops(&output.script_pubkey, false);
+        }
+        n
+    }
+
+    fn get_p2sh_sigop_count(tx: &Transaction, previous_outputs: &[&TxOut]) -> usize {
+        #[cfg(not(feature = "liquid"))]
+        if tx.is_coinbase() {
+            return 0;
+        }
+        #[cfg(feature = "liquid")]
+        if tx.is_coinbase() {
+            return 0;
+        }
+        let mut n = 0;
+        for (input, prevout) in tx.input.iter().zip(previous_outputs.iter()) {
+            if prevout.script_pubkey.is_p2sh() {
+                if let Some(Ok(script::Instruction::PushBytes(redeem))) =
+                    input.script_sig.instructions().last()
+                {
+                    #[cfg(not(feature = "liquid"))]
+                    let script = script::Script::from_bytes(redeem.as_bytes());
+                    #[cfg(feature = "liquid")]
+                    let script = script::Script::from(redeem.to_vec());
+                    #[allow(clippy::needless_borrow)]
+                    {
+                        n += count_sigops(&script, true);
+                    }
+                }
+            }
+        }
+        n
+    }
+
+    fn get_witness_sigop_count(tx: &Transaction, previous_outputs: &[&TxOut]) -> usize {
+        let mut n = 0;
+
+        #[inline]
+        fn is_push_only(script: &script::Script) -> bool {
+            for inst in script.instructions() {
+                match inst {
+                    Err(_) => return false,
+                    Ok(Instruction::Op(_)) => return false,
+                    Ok(Instruction::PushBytes(_)) => {}
+                }
+            }
+            true
+        }
+
+        #[inline]
+        fn last_pushdata(script: &script::Script) -> Option<&[u8]> {
+            match script.instructions().last() {
+                #[cfg(not(feature = "liquid"))]
+                Some(Ok(Instruction::PushBytes(bytes))) => Some(bytes.as_bytes()),
+                #[cfg(feature = "liquid")]
+                Some(Ok(Instruction::PushBytes(bytes))) => Some(bytes),
+                _ => None,
+            }
+        }
+
+        #[inline]
+        fn count_with_prevout(
+            prevout: &TxOut,
+            script_sig: &script::Script,
+            witness: &Witness,
+        ) -> usize {
+            let mut n = 0;
+
+            let script_owned;
+            let script: &script::Script = if prevout.script_pubkey.is_witness_program() {
+                &prevout.script_pubkey
+            } else if prevout.script_pubkey.is_p2sh()
+                && is_push_only(script_sig)
+                && !script_sig.is_empty()
+            {
+                #[cfg(not(feature = "liquid"))]
+                {
+                    script_owned =
+                        script::ScriptBuf::from(last_pushdata(script_sig).unwrap().to_vec());
+                }
+                #[cfg(feature = "liquid")]
+                {
+                    script_owned =
+                        script::Script::from(last_pushdata(script_sig).unwrap().to_vec());
+                }
+                &script_owned
+            } else {
+                return 0;
+            };
+
+            #[cfg(not(feature = "liquid"))]
+            if script.is_p2wsh() {
+                let bytes = script.as_bytes();
+                n += sig_ops(witness, bytes[0], &bytes[2..]);
+            } else if script.is_p2wpkh() {
+                n += 1;
+            }
+            #[cfg(feature = "liquid")]
+            if script.is_v0_p2wsh() {
+                let bytes = script.as_bytes();
+                n += sig_ops(witness, bytes[0], &bytes[2..]);
+            } else if script.is_v0_p2wpkh() {
+                n += 1;
+            }
+            n
+        }
+
+        for (input, prevout) in tx.input.iter().zip(previous_outputs.iter()) {
+            n += count_with_prevout(prevout, &input.script_sig, &input.witness);
+        }
+        n
+    }
+
+    /// Get the sigop cost for this transaction.
+    fn get_sigop_cost(
+        tx: &Transaction,
+        previous_outputs: &[&TxOut],
+        verify_p2sh: bool,
+        verify_witness: bool,
+    ) -> Result<usize, script::Error> {
+        let mut n_sigop_cost = get_legacy_sigop_count(tx) * 4;
+        #[cfg(not(feature = "liquid"))]
+        if tx.is_coinbase() {
+            return Ok(n_sigop_cost);
+        }
+        #[cfg(feature = "liquid")]
+        if tx.is_coinbase() || tx.input.iter().any(|input| input.is_pegin) {
+            return Ok(n_sigop_cost);
+        }
+        if tx.input.len() != previous_outputs.len() {
+            return Err(script::Error::EarlyEndOfScript);
+        }
+        if verify_witness && !verify_p2sh {
+            return Err(script::Error::EarlyEndOfScript);
+        }
+        if verify_p2sh {
+            n_sigop_cost += get_p2sh_sigop_count(tx, previous_outputs) * 4;
+        }
+        if verify_witness {
+            n_sigop_cost += get_witness_sigop_count(tx, previous_outputs);
+        }
+
+        Ok(n_sigop_cost)
+    }
+
+    /// Get sigops for the Witness
+    ///
+    /// witness_version is the raw opcode. OP_0 is 0, OP_1 is 81, etc.
+    #[allow(clippy::redundant_closure)]
+    fn sig_ops(witness: &Witness, witness_version: u8, witness_program: &[u8]) -> usize {
+        #[cfg(feature = "liquid")]
+        let last_witness = witness.script_witness.last();
+        #[cfg(not(feature = "liquid"))]
+        let last_witness = witness.last();
+        match (witness_version, witness_program.len()) {
+            (0, 20) => 1,
+            (0, 32) => {
+                #[cfg(not(feature = "liquid"))]
+                {
+                    #[allow(clippy::needless_borrow)]
+                    last_witness
+                        .map(|sl| script::Script::from_bytes(sl))
+                        .map(|s| count_sigops(s, true))
+                        .unwrap_or_default()
+                }
+                #[cfg(feature = "liquid")]
+                {
+                    last_witness
+                        .map(|sl| script::Script::from(sl.clone()))
+                        .map(|s| count_sigops(&s, true))
+                        .unwrap_or_default()
+                }
+            }
+            _ => 0,
+        }
+    }
+}

start

@@ -0,0 +1,274 @@
+#!/usr/bin/env zsh
+
+# initialize variables
+DAEMON=bitcoin
+NETWORK=mainnet
+FEATURES=default
+DB_FOLDER=/electrs
+ASSET_DB_ARGS=()
+NODENAME=$(hostname|cut -d . -f1)
+LOCATION=$(hostname|cut -d . -f2)
+USAGE="Usage: $0 (mainnet|testnet|signet|liquid|liquidtestnet) [popular-scripts]"
+
+# load rust if necessary
+if [ -e "${HOME}/.cargo/env" ];then
+	source "${HOME}/.cargo/env"
+	export PATH="${HOME}/.cargo/bin:${PATH}"
+fi
+
+# which OS?
+case "$(uname -s)" in
+	FreeBSD)
+		OS=FreeBSD
+		NPROC=$(sysctl -n hw.ncpu)
+		export CC=/usr/local/bin/clang17
+		export CXX=/usr/local/bin/clang++17
+		export CPP=/usr/local/bin/clang-cpp17
+		export RUSTFLAGS="-C linker=clang17"
+	;;
+	Darwin)
+		OS=Darwin
+		NPROC=$(sysctl -n hw.ncpu)
+	;;
+	Linux)
+		OS=Linux
+		NPROC=$(grep -c proc /proc/cpuinfo)
+	;;
+	*)
+		OS=Unknown
+		NPROC=4
+	;;
+esac
+
+# which network?
+case "${1}" in
+	mainnet)
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="20 4 * * *"
+	;;
+	testnet)
+		NETWORK=testnet
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="2 4 * * *"
+	;;
+	testnet4)
+		NETWORK=testnet4
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="17 4 * * *"
+	;;
+	signet)
+		NETWORK=signet
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="9 4 * * *"
+	;;
+	liquid)
+		DAEMON=elements
+		NETWORK=liquid
+		FEATURES=liquid
+		ASSET_DB_ARGS=(--asset-db-path /elements/asset_registry_db)
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="12 4 * * *"
+	;;
+	liquidtestnet)
+		DAEMON=elements
+		NETWORK=liquidtestnet
+		FEATURES=liquid
+		ASSET_DB_ARGS=(--asset-db-path /elements/asset_registry_testnet_db)
+		THREADS=$((NPROC / 8))
+		CRONJOB_TIMING="17 4 * * *"
+	;;
+	*)
+		echo "${USAGE}"
+		exit 1
+	;;
+esac
+
+# Run the popular address txt file generator before each run
+POPULAR_SCRIPTS_FOLDER="${HOME}/popular-scripts/${NETWORK}"
+POPULAR_SCRIPTS_FILE_RAW="${POPULAR_SCRIPTS_FOLDER}/popular-scripts-raw.txt"
+POPULAR_SCRIPTS_FILE="${POPULAR_SCRIPTS_FOLDER}/popular-scripts.txt"
+
+# This function runs the job for generating the popular scripts text file for the precache arg
+generate_popular_scripts() {
+	mkdir -p "${POPULAR_SCRIPTS_FOLDER}"
+
+	## Use nproc * 4 threads to generate the txt file (lots of iowait, so 2x~4x core count is ok)
+	## Only pick up addresses with 101 history events or more
+	## (Without lowering MIN_HISTORY_ITEMS_TO_CACHE this is the lowest we can go)
+	## It prints out progress to STDERR
+	echo "[*] Generating popular-scripts using ${THREADS} threads..."
+	cd "${HOME}/electrs"
+	HIGH_USAGE_THRESHOLD=101 \
+	JOB_THREAD_COUNT=${THREADS} \
+	nice cargo run \
+		--release \
+		--bin popular-scripts \
+		--features "${FEATURES}" \
+		-- \
+		--network "${NETWORK}" \
+		--db-dir "${DB_FOLDER}" \
+		> "${POPULAR_SCRIPTS_FILE_RAW}"
+
+	## Only overwrite the existing file if the popular-scripts cargo run succeeded
+	if [ "$?" = "0" ];then
+		## Sorted and deduplicated just in case
+		echo "Sorting popular scripts for final results..."
+		sort "${POPULAR_SCRIPTS_FILE_RAW}" | uniq > "${POPULAR_SCRIPTS_FILE}"
+	fi
+
+	rm "${POPULAR_SCRIPTS_FILE_RAW}"
+}
+
+# This function is for inserting the cronjob for generating the popular scripts
+CRONJOB_CMD="\"${HOME}/electrs/start\" \"${NETWORK}\" popular-scripts"
+echo "${CRONJOB_TIMING} ${CRONJOB_CMD}"
+
+case "${2}" in
+	popular-scripts)
+		echo "[*] Only generate popular-scripts, then exit"
+		generate_popular_scripts
+		exit 0
+	;;
+	version)
+		echo "[*] Only print versions, then exit"
+		cargo run --bin electrs --release -- --version
+		cargo run --bin popular-scripts --release -- --version
+		exit 0
+	;;
+	"")
+		# If the 2nd arg isn't passed, just run the normal electrs script as-is
+	;;
+	*)
+		echo "${USAGE}"
+		exit 1
+	;;
+esac
+
+# run in loop in case of crash
+until false
+do
+	# reset CWD
+	cd "${HOME}/electrs"
+
+	# disable making electrs.core files
+	ulimit -c 0
+
+	# prepare run-time variables
+	UTXOS_LIMIT=500
+	ELECTRUM_TXS_LIMIT=500
+	ELECTRUM_MAX_LINE_SIZE=1048576 # 1 MiB
+	ELECTRUM_MAX_SUBSCRIPTIONS=100
+	ELECTRUM_MAX_CLIENTS=1000
+	MAIN_LOOP_DELAY=500
+	DAEMON_CONF="${HOME}/${DAEMON}.conf"
+	HTTP_SOCKET_FILE="${HOME}/socket/esplora-${DAEMON}-${NETWORK}"
+	RPC_SOCKET_FILE="${HOME}/socket/electrum-${DAEMON}-${NETWORK}"
+
+	# get RPC credentials from bitcoin.conf or elements.conf directly
+	echo "[*] Getting RPC credentials from ${DAEMON_CONF}"
+	RPC_USER=$(grep 'rpcuser=' "${DAEMON_CONF}"|cut -d = -f2|head -1)
+	RPC_PASS=$(grep 'rpcpassword=' "${DAEMON_CONF}"|cut -d = -f2|head -1)
+
+	# override limits based on hostname
+	if [ "${NODENAME}" = "node201" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+		MAIN_LOOP_DELAY=14000
+	fi
+	if [ "${NODENAME}" = "node204" ] && [ "${LOCATION}" = "sg1" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node204" ] && [ "${LOCATION}" = "hnl" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node206" ] && [ "${LOCATION}" = "tk7" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node211" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node212" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node213" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NODENAME}" = "node214" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${NETWORK}" = "testnet4" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+	if [ "${LOCATION}" = "fmt" ];then
+		UTXOS_LIMIT=9000
+		ELECTRUM_TXS_LIMIT=9000
+		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
+		ELECTRUM_MAX_SUBSCRIPTIONS=100000
+		ELECTRUM_MAX_CLIENTS=10000
+	fi
+
+	if [ ! -e "${POPULAR_SCRIPTS_FILE}" ];then
+		generate_popular_scripts
+	fi
+
+	# Run the electrs process (Note: db-dir is used in both commands)
+	nice cargo run \
+		--release \
+		--bin electrs \
+		--features "${FEATURES}" \
+		-- \
+		--network "${NETWORK}" \
+		--daemon-dir "${HOME}" \
+		--db-dir "${DB_FOLDER}" \
+		"${ASSET_DB_ARGS[@]}" \
+		--main-loop-delay "${MAIN_LOOP_DELAY}" \
+		--rpc-socket-file "${RPC_SOCKET_FILE}" \
+		--http-socket-file "${HTTP_SOCKET_FILE}" \
+		--precache-scripts "${POPULAR_SCRIPTS_FILE}" \
+		--precache-threads "${THREADS}" \
+		--cookie "${RPC_USER}:${RPC_PASS}" \
+		--cors '*' \
+		--address-search \
+		--utxos-limit "${UTXOS_LIMIT}" \
+		--electrum-txs-limit "${ELECTRUM_TXS_LIMIT}" \
+		--electrum-max-line-size "${ELECTRUM_MAX_LINE_SIZE}" \
+		--electrum-max-subscriptions "${ELECTRUM_MAX_SUBSCRIPTIONS}" \
+		--electrum-max-clients "${ELECTRUM_MAX_CLIENTS}" \
+		-vv
+	sleep 1
+done