Run 'cargo fmt'

This adds the following new endpoints:

- GET /scripthash/:scripthash
- GET /scripthash/:scripthash/txs
- GET /scripthash/:scripthash/utxo

(cherry picked from commit 16891ae00e)

.ackrc

@@ -1,2 +0,0 @@
---ignore-dir=target
---ignore-dir=db

.dockerignore

@@ -1 +0,0 @@
-Dockerfile

.editorconfig

@@ -1,6 +0,0 @@
-# see https://editorconfig.org for more options, and setup instructions for yours editor
-
-[*]
-indent_style = space
-indent_size = 4
-

.github/actions/ci-rust-setup/action.yml

@@ -1,50 +0,0 @@
-name: CI Rust Setup
-description: 'Sets up the environment for Rust jobs during CI workflow'
-
-inputs:
-  cache-name:
-    description: 'Name of cache artifacts (same name is same cache key) empty to disable cache'
-    required: false
-  targets:
-    description: 'A comma separated list of extra targets you want to install'
-    required: false
-  components:
-    description: 'A comma separated list of extra components you want to install'
-    required: false
-  toolchain:
-    description: 'The toolchain to use. If not specified, the rust-toolchain file will be used'
-    required: false
-
-runs:
-  using: composite
-  steps:
-    - name: Get toolchain from input OR rust-toolchain file
-      id: gettoolchain
-      shell: bash
-      run: |-
-        RUST_TOOLCHAIN="${{ inputs.toolchain }}"
-        if [ ! -f rust-toolchain ] && [ -z "${RUST_TOOLCHAIN}" ]; then
-          echo "***ERROR*** NEED toolchain INPUT OR rust-toolchain FILE IN ROOT OF REPOSITORY" >&2
-          exit 1
-        fi
-        if [ -z "${RUST_TOOLCHAIN}" ]; then
-          RUST_TOOLCHAIN="$(cat rust-toolchain)"
-        fi
-        echo "toolchain=\"${RUST_TOOLCHAIN}\"" >> $GITHUB_OUTPUT
-    - name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
-      id: toolchain
-      # Commit date is Nov 18, 2024
-      uses: dtolnay/rust-toolchain@315e265cd78dad1e1dcf3a5074f6d6c47029d5aa
-      with:
-        toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
-        targets: ${{ inputs.targets }}
-        components: ${{ inputs.components }}
-    - name: Cache dependencies
-      uses: actions/cache@v3
-      if: inputs.cache-name != ''
-      with:
-        path: |
-          ~/.cargo/registry
-          ~/.cargo/git
-          target
-        key: ${{ runner.os }}-cargo-${{ inputs.cache-name }}-${{ steps.toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock') }}

.github/workflows/ci.yml

@@ -1,107 +0,0 @@
-on:
-  pull_request:
-  push:
-    branches:
-      - mempool
-
-
-name: Compile Check and Lint
-
-jobs:
-  check:
-    name: Compile Check
-    runs-on: mempool-ci
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust
-        uses: './.github/actions/ci-rust-setup'
-        with:
-          cache-name: dev
-      - run: cargo check --all-features
-
-  fmt:
-    name: Formatter
-    runs-on: mempool-ci
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust
-        uses: './.github/actions/ci-rust-setup'
-        with:
-          components: rustfmt
-      - run: cargo fmt --all -- --check
-
-  test:
-    name: Run Tests
-    runs-on: mempool-ci
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust
-        uses: './.github/actions/ci-rust-setup'
-        with:
-          cache-name: test
-      - run: cargo test --lib --all-features
-
-  compile-freebsd:
-    runs-on: mempool-ci
-    name: Run Compile Checks in FreeBSD
-    env:
-      FREEBSD_VER: "14.3"
-    steps:
-    - uses: actions/checkout@v4
-    - name: Cache dependencies for FreeBSD
-      uses: actions/cache@v3
-      with:
-        path: |
-          .cargohome/registry
-          .cargohome/git
-          target
-        key: freebsd-${{ env.FREEBSD_VER }}-cargo-checks-${{ hashFiles('**/Cargo.lock') }}
-    - name: Compile Checks in FreeBSD
-      uses: vmactions/freebsd-vm@v1
-      with:
-        usesh: true
-        release: "${{ env.FREEBSD_VER }}"
-        arch: amd64
-        prepare: |
-          mkdir -p ~/.cargo/
-          mkdir -p ./.cargohome/registry/
-          mkdir -p ./.cargohome/git/
-          mv ./.cargohome/registry ~/.cargo/
-          mv ./.cargohome/git ~/.cargo/
-          rm -rf ./.cargohome
-          pkg install -y git rsync gmake llvm rust rocksdb cmake
-
-        run: |
-          cargo check --no-default-features
-          cargo check -F liquid
-          cargo check -F electrum-discovery
-          cargo check -F electrum-discovery,liquid
-          cargo build --release --bin electrs
-          rm -rf ./.cargohome
-          mkdir -p ~/.cargo/registry/
-          mkdir -p ~/.cargo/git/
-          mkdir -p ./.cargohome/
-          mv ~/.cargo/registry ./.cargohome/
-          mv ~/.cargo/git ./.cargohome/
-
-  clippy:
-    name: Linter
-    runs-on: mempool-ci
-    needs: [check]
-    strategy:
-      matrix: # Try all combinations of features. Some times weird things appear.
-        features: [
-          '',
-          '-F electrum-discovery',
-          '-F liquid',
-          '-F electrum-discovery,liquid',
-        ]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust
-        uses: './.github/actions/ci-rust-setup'
-        with:
-          cache-name: dev
-          components: clippy
-      - name: Clippy with Features = ${{ matrix.features }}
-        run: cargo clippy ${{ matrix.features }} -- -D warnings

.github/workflows/on-tag.yml

@@ -1,77 +0,0 @@
-name: Docker build on tag
-env:
-  DOCKER_CLI_EXPERIMENTAL: enabled
-  TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
-  DOCKER_BUILDKIT: 0
-  COMPOSE_DOCKER_CLI_BUILD: 0
-
-on:
-  push:
-    tags:
-      - v[0-9]+.[0-9]+.[0-9]+
-      - v[0-9]+.[0-9]+.[0-9]+-*
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    runs-on: mempool-ci
-    timeout-minutes: 120
-    name: Build and push to DockerHub
-    strategy:
-      max-parallel: 1
-      matrix:
-        include:
-          - image: electrs
-            cargo_extra_args: ""
-          - image: electrs-liquid
-            cargo_extra_args: "--features liquid"
-    steps:
-      - name: Set env variables
-        run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
-
-      - name: Show set environment variables
-        run: |
-          printf "      TAG: %s\n"  "$TAG"
-
-      - name: Add SHORT_SHA env property with commit short sha
-        run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
-
-      - name: Login to Docker for building
-        run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
-
-      - name: Checkout project
-        uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        id: qemu
-
-      - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@v3
-        id: buildx
-
-      - name: Available platforms
-        run: echo ${{ steps.buildx.outputs.platforms }}
-
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        id: cache
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx
-          restore-keys: |
-            ${{ runner.os }}-buildx
-
-      - name: Run Docker buildx against tag
-        run: |
-          docker buildx build \
-          --cache-from "type=local,src=/tmp/.buildx-cache" \
-          --cache-to "type=local,dest=/tmp/.buildx-cache" \
-          --platform linux/amd64,linux/arm64 \
-          --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.image }}:$TAG \
-          --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.image }}:latest \
-          --output "type=registry" . \
-          --build-arg commitHash=$SHORT_SHA \
-          --build-arg CARGO_EXTRA_ARGS="${{ matrix.cargo_extra_args }}"

.github/workflows/project-review-status.yml

@@ -1,18 +0,0 @@
-name: Project Board Automation
-
-on:
-  pull_request:
-    types: [review_requested]
-  issues:
-    types: [opened]
-
-jobs:
-  project-automation:
-    uses: mempool/.github/.github/workflows/project-board-automation.yml@master
-    with:
-      project-number: 8
-    secrets:
-      PROJECT_TOKEN: ${{ secrets.PROJECT_TOKEN }}
-      PROJECT_ID: ${{ secrets.PROJECT_ID }}
-      STATUS_FIELD_ID: ${{ secrets.STATUS_FIELD_ID }}
-      REVIEW_NEEDED_OPTION_ID: ${{ secrets.REVIEW_NEEDED_OPTION_ID }}

.gitignore

@@ -4,5 +4,5 @@ target
 *.sublime*
 *~
 *.pyc
-.vscode
-*.core
+.idea/
+start.sh

.travis.yml

@@ -13,4 +13,3 @@ script:
   - cargo check --all
   - cargo build --all
   - cargo test --all
-  - cargo build --features "liquid" --all

AGENTS.md

@@ -1,19 +0,0 @@
-# electrs
-
-## Rules
-
-1. You are an expert Rust developer.
-2. You are an expert Bitcoin developer.
-3. If you are unsure of a change, ask the developer to make a choice proactively.
-
-## Before testing
-
-- Run cargo fmt (from root)
-  - command: `cargo fmt`
-
-## Testing
-
-- Run the checks script
-  - `./scripts/checks.sh`
-- Run with tests only when a test is added or changed
-  - `INCLUDE_TESTS=1 ./scripts/checks.sh`

Cargo.toml

@@ -1,78 +1,52 @@
 [package]
-name = "mempool-electrs"
-version = "3.4.0-dev"
-authors = [
-    "Roman Zeyde <me@romanzey.de>",
-    "Nadav Ivgi <nadav@shesek.info>",
-    "wiz <j@wiz.biz>",
-    "junderw <jonathan.underwood4649@gmail.com>"
-]
+name = "electrs"
+version = "0.4.1"
+authors = ["Roman Zeyde <me@romanzey.de>"]
 description = "An efficient re-implementation of Electrum Server in Rust"
 license = "MIT"
-homepage = "https://github.com/mempool/electrs"
-repository = "https://github.com/mempool/electrs"
-publish = false
+homepage = "https://github.com/romanz/electrs"
+repository = "https://github.com/romanz/electrs"
 keywords = ["bitcoin", "electrum", "server", "index", "database"]
+documentation = "https://docs.rs/electrs/"
 readme = "README.md"
-edition = "2018"
-
-[lib]
-name = "electrs"
-
-[features]
-default = []
-liquid = ["elements"]
-electrum-discovery = ["electrum-client"]
 
 [dependencies]
-arrayref = "0.3.6"
-base64 = "0.13.0"
-bincode-do-not-use-directly = { version = "1.3.1", package = "bincode" }
-bitcoin = { version = "0.32.8", features = [ "serde" ] }
-bounded-vec-deque = "0.1.1"
-clap = "2.33.3"
-crossbeam-channel = "0.5.0"
-dirs = "4.0.0"
-elements = { version = "0.26.1", features = [ "serde" ], optional = true }
-error-chain = "0.12.4"
-glob = "0.3"
-hex = "0.4.2"
-itertools = "0.10"
-lazy_static = "1.3.0"
+arrayref = "0.3"
+base64 = "0.9"
+bincode = "1.0"
+bitcoin-bech32 = "0.8.0"
+chan = "0.1"
+chan-signal = "0.3"
+clap = "2.31"
+dirs = "1.0"
+error-chain = "0.12"
+glob = "0.2"
+hex = "0.3"
 libc = "0.2"
-log = "0.4.11"
-socket2 = { version = "0.4", features = ["all"] }
-num_cpus = "1.12.0"
-page_size = "0.4.2"
-prometheus = "0.13"
-ppp = "2.3.0"
-rayon = "1.5.0"
-rocksdb = "0.24.0"
-serde = "1.0.118"
-serde_derive = "1.0.118"
-serde_json = "1.0.60"
-sha2 = "0.10.7"
-signal-hook = "0.3"
-stderrlog = "0.5.0"
-time = { version = "0.3", features = ["formatting"] }
-tiny_http = "0.11"
-url = "2.2.0"
-hyper = "0.14"
-hyperlocal = "0.8"
-# close to same tokio version as dependent by hyper v0.14 and hyperlocal 0.8 -- things can go awry if they mismatch
-tokio = { version = "1", features = ["sync", "macros"] }
+log = "0.4"
+lru = "0.1"
+num_cpus = "1.0"
+page_size = "0.4"
+prometheus = "0.4"
+rocksdb = "0.10.1"
+rust-crypto = "0.2"
+secp256k1 = "0.11"
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+stderrlog = "0.4.1"
+sysconf = ">=0.3.4"
+time = "0.1"
+tiny_http = "0.6"
+hyper = "0.12"
+url = "1.0"
+lru-cache = "0.1.1"
 
-# optional dependencies for electrum-discovery
-electrum-client = { version = "0.24.1", optional = true }
+[dependencies.elements]
+git = "https://github.com/ElementsProject/rust-elements/"
+rev = "78c1e29809b7420041b1259711f0a2934211a0d3" # from master branch
+features = ["serde-feature"] # Doesn't look to work by now
 
-[dev-dependencies]
-tempfile = "3.0"
+[dependencies.bitcoin]
+version = "0.15.1"
 
-[profile.release]
-lto = true
-panic = 'abort'
-codegen-units = 1
-
-[patch.crates-io.electrum-client]
-git = "https://github.com/mempool/rust-electrum-client"
-rev = "4bbfc612d594fe23282c439d4bdc446cff01ba1c" # 0.24.1/add-peer branch

Dockerfile

@@ -1,30 +1,21 @@
-FROM debian:bookworm-slim AS base
+FROM rust:latest
 
-ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
+RUN apt-get update
+RUN apt-get install -y clang cmake
 
-RUN apt update -qy && \
-    apt install -qy librocksdb-dev curl
+RUN cargo install electrs
 
-FROM base as build
+RUN adduser --disabled-login --system --shell /bin/false --uid 1000 user
 
-RUN apt install -qy git clang cmake
-
-ENV RUSTUP_HOME=/rust
-ENV CARGO_HOME=/cargo 
-ENV PATH=/cargo/bin:/rust/bin:$PATH
-
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-
-WORKDIR /build
-COPY . .
-
-ARG CARGO_EXTRA_ARGS=""
-RUN cargo build --release --bin electrs ${CARGO_EXTRA_ARGS}
-
-FROM base as deploy
-
-COPY --from=build /build/target/release/electrs /bin/electrs
+USER user
+WORKDIR /home/user
 
+# Electrum RPC
 EXPOSE 50001
 
-ENTRYPOINT ["/bin/electrs"]
+# Prometheus monitoring
+EXPOSE 4224
+
+STOPSIGNAL SIGINT
+
+CMD ["electrs", "-vvvv", "--timestamp"]

README.md

@@ -1,83 +1,8 @@
-# Mempool - Electrs backend API
+# Esplora - Electrs backend API
 
-A block chain index engine and HTTP API written in Rust based on [romanz/electrs](https://github.com/romanz/electrs) and [Blockstream/electrs](https://github.com/Blockstream/electrs).
+A block chain index engine and HTTP API written in Rust based on [romanz/electrs](https://github.com/romanz/electrs).
 
-Used as the backend for the [mempool block explorer](https://github.com/mempool/mempool) powering [mempool.space](https://mempool.space/).
+The `liquid_e` branch supports Liquid and other Elements-based chains,
+including confidential transactions, peg-in/out and multi-asset.
 
-API documentation [is available here](https://mempool.space/docs/api/rest).
-
-Documentation for the database schema and indexing process [is available here](doc/schema.md).
-
-### Installing & indexing
-
-Install Rust, Bitcoin Core (no `txindex` needed) and the `clang` and `cmake` packages, then:
-
-```bash
-$ git clone https://github.com/mempool/electrs && cd electrs
-$ git checkout mempool
-$ cargo run --release --bin electrs -- -vvvv --daemon-dir ~/.bitcoin
-
-# Or for liquid:
-$ cargo run --features liquid --release --bin electrs -- -vvvv --network liquid --daemon-dir ~/.liquid
-```
-
-See [electrs's original documentation](https://github.com/romanz/electrs/blob/master/doc/usage.md) for more detailed instructions.
-Note that our indexes are incompatible with electrs's and has to be created separately.
-
-The indexes require 1.3TB of storage after running compaction (as of October 2023), but you'll need to have
-free space of about double that available during the index compaction process.
-Creating the indexes should take a few hours on a beefy machine with high speed NVMe SSD(s).
-
-### Light mode
-
-For personal or low-volume use, you may set `--lightmode` to reduce disk storage requirements
-by roughly 50% at the cost of slower and more expensive lookups.
-
-With this option set, raw transactions and metadata associated with blocks will not be kept in rocksdb
-(the `T`, `X` and `M` indexes),
-but instead queried from bitcoind on demand.
-
-### Notable changes from Electrs:
-
-- HTTP REST API in addition to the Electrum JSON-RPC protocol, with extended transaction information
-  (previous outputs, spending transactions, script asm and more).
-
-- Extended indexes and database storage for improved performance under high load:
-
-  - A full transaction store mapping txids to raw transactions is kept in the database under the prefix `t`.
-  - An index of all spendable transaction outputs is kept under the prefix `O`.
-  - An index of all addresses (encoded as string) is kept under the prefix `a` to enable by-prefix address search.
-  - A map of blockhash to txids is kept in the database under the prefix `X`.
-  - Block stats metadata (number of transactions, size and weight) is kept in the database under the prefix `M`.
-
-  With these new indexes, bitcoind is no longer queried to serve user requests and is only polled
-  periodically for new blocks and for syncing the mempool.
-
-- Support for Liquid and other Elements-based networks, including CT, peg-in/out and multi-asset.
-  (requires enabling the `liquid` feature flag using `--features liquid`)
-
-### CLI options
-
-In addition to electrs's original configuration options, a few new options are also available:
-
-- `--http-addr <addr:port>` - HTTP server address/port to listen on (default: `127.0.0.1:3000`).
-- `--lightmode` - enable light mode (see above)
-- `--cors <origins>` - origins allowed to make cross-site request (optional, defaults to none).
-- `--address-search` - enables the by-prefix address search index.
-- `--index-unspendables` - enables indexing of provably unspendable outputs.
-- `--utxos-limit <num>` - maximum number of utxos to return per address.
-- `--electrum-txs-limit <num>` - maximum number of txs to return per address in the electrum server (does not apply for the http api).
-- `--electrum-banner <text>` - welcome banner text for electrum server.
-
-Additional options with the `liquid` feature:
-- `--parent-network <network>` - the parent network this chain is pegged to.
-
-Additional options with the `electrum-discovery` feature:
-- `--electrum-hosts <json>` - a json map of the public hosts where the electrum server is reachable, in the [`server.features` format](https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server.features).
-- `--electrum-announce` - announce the electrum server on the electrum p2p server discovery network.
-
-See `$ cargo run --bin electrs -- --help` for the full list of options.
-
-## License
-
-MIT
+See [the main README in `bitcoin_e`](https://github.com/Blockstream/electrs) for more details.

build.rs

@@ -1,57 +0,0 @@
-use std::{path::Path, process::Command};
-
-fn main() {
-    // Specify re-run conditions
-
-    // 1. Rerun build script if we pass a new GIT_HASH
-    println!("cargo:rerun-if-env-changed=GIT_HASH");
-
-    // 2. Only do git based reruns if git directory exists
-    if Path::new(".git").exists() {
-        // If we change the branch, rerun
-        println!("cargo:rerun-if-changed=.git/HEAD");
-        if let Ok(r) = std::fs::read_to_string(".git/HEAD") {
-            if let Some(stripped) = r.strip_prefix("ref: ") {
-                // If the HEAD is detached it will be a commit hash
-                // so the HEAD changed directive above will pick it up,
-                // otherwise it will point to a ref in the refs directory
-                println!("cargo:rerun-if-changed=.git/{}", stripped);
-            }
-        }
-    }
-
-    // Getting git hash
-
-    // Don't fetch git hash if it's already in the ENV
-    let existing = std::env::var("GIT_HASH").unwrap_or_else(|_| String::new());
-    if !existing.is_empty() {
-        return;
-    }
-
-    // Get git hash from git and don't do anything if the command fails
-    if let Some(rev_parse) = cmd("git", &["rev-parse", "--short", "HEAD"]) {
-        // Add (dirty) to the GIT_HASH if the git status isn't clean
-        // This includes untracked files
-        let dirty = cmd("git", &["status", "--short"]).expect("git command works");
-
-        // Ignore Dockerfile deletion as it is expected in Docker buildx builds
-        let git_hash = if dirty.is_empty() || dirty.trim() == "D Dockerfile" {
-            rev_parse
-        } else {
-            format!("{}(dirty)", rev_parse.trim())
-        };
-
-        println!("cargo:rustc-env=GIT_HASH={}", git_hash.trim());
-    }
-}
-
-// Helper function, Command is verbose...
-fn cmd(name: &str, args: &[&str]) -> Option<String> {
-    Command::new(name).args(args).output().ok().and_then(|o| {
-        if o.status.success() {
-            String::from_utf8(o.stdout).ok()
-        } else {
-            None
-        }
-    })
-}

doc/schema.md

@@ -1,93 +1,31 @@
 # Index Schema
 
-The index is stored as three RocksDB databases:
+The index is stored at a single RocksDB database using the following schema:
 
-- `txstore`
-- `history`
-- `cache`
+## Transaction outputs' index
 
-### Indexing process
+Allows efficiently finding all funding transactions for a specific address:
 
-The indexing is done in the two phase, where each can be done concurrently within itself.
-The first phase populates the `txstore` database, the second phase populates the `history` database.
+|  Code  | Script Hash Prefix   | Funding TxID Prefix   |   |
+| ------ | -------------------- | --------------------- | - |
+| `b'O'` | `SHA256(script)[:8]` | `txid[:8]`            |   |
 
-NOTE: in order to construct the history rows for spending inputs in phase #2, we rely on having the transactions being processed at phase #1, so they can be looked up efficiently (using parallel point lookups).
+## Transaction inputs' index
 
-After the indexing is completed, both funding and spending are indexed as independent rows under `H{scripthash}`, so that they can be queried in-order in one go.
+Allows efficiently finding spending transaction of a specific output:
 
-### `txstore`
+|  Code  | Funding TxID Prefix  | Funding Output Index  | Spending TxID Prefix  |   |
+| ------ | -------------------- | --------------------- | --------------------- | - |
+| `b'I'` | `txid[:8]`           | `uint16`              | `txid[:8]`            |   |
 
-Each block results in the following new rows:
 
- * `"B{blockhash}" → "{header}"`
+## Full Transaction IDs
 
- * `"X{blockhash}" → "{txids}"` (list of txids included in the block)
+In order to save storage space, we store the full transaction IDs once, and use their 8-byte prefixes for the indexes above.
 
- * `"M{blockhash}" → "{metadata}"` (block weight, size and number of txs)
+|  Code  | Transaction ID    |   | Confirmed height   |
+| ------ | ----------------- | - | ------------------ |
+| `b'T'` | `txid` (32 bytes) |   | `uint32`           |
 
- * `"D{blockhash}" → ""` (signifies the block is done processing)
-
-Each transaction results in the following new rows:
-
- * `"T{txid}" → "{serialized-transaction}"`
-
- * `"C{txid}{confirmed-blockhash}" → ""` (a list of blockhashes where `txid` was seen to be confirmed)
-
-Each output results in the following new row:
-
- * `"O{txid}{vout}" → "{scriptpubkey}{value}"`
-
-When the indexer is synced up to the tip of the chain, the hash of the tip is saved as following:
-
- * `"t" →  "{blockhash}"`
-
-### `history`
-
-Each funding output (except for provably unspendable ones when `--index-unspendables` is not enabled) results in the following new rows (`H` is for history, `F` is for funding):
-
- * `"H{funding-scripthash}{funding-height}F{funding-txid:vout}{value}" → ""`
- * `"a{funding-address-str}" → ""` (for prefix address search, only saved when `--address-search` is enabled)
-
-Each spending input (except the coinbase) results in the following new rows (`S` is for spending):
-
- * `"H{funding-scripthash}{spending-height}S{spending-txid:vin}{funding-txid:vout}{value}" → ""`
-
- * `"S{funding-txid:vout}{spending-txid:vin}" → ""`
-
-#### Elements only
-
-Assets (re)issuances results in the following new rows (only for user-issued assets):
-
- * `"i{asset-id}" → "{issuing-txid:vin}{prev-txid:vout}{issuance}{reissuance_token}"`
- * `"I{asset-id}{issuance-height}I{issuing-txid:vin}{is_reissuance}{amount}{tokens}" → ""`
-
-Peg-ins/peg-outs results in the following new rows (only for the native asset, typically L-BTC):
-
- * `"I{asset-id}{pegin-height}F{pegin-txid:vin}{value}" → ""`
- * `"I{asset-id}{pegout-height}F{pegout-txid:vout}{value}" → ""`
-
-Every burn (unspendable output) results in the following new row (both user-issued and native):
-
- * `"I{asset-id}{burn-height}F{burning-txid:vout}{value}" → ""`
-
-### `cache`
-
-Holds a cache for aggregated stats and unspent TXOs of scripthashes.
-
-The cache is created on-demand, the first time the scripthash is requested by a user.
-
-The cached data is kept next to the `blockhash` the cache is up-to-date for.
-When requesting data, the cache is updated with the new history rows added since the `blockhash`.
-If the `blockhash` was since orphaned, the cache is removed and re-computed.
-
- * `"A{scripthash}" → "{stats}{blockhash}"` (where `stats` is composed of `tx_count`, `funded_txo_{count,sum}` and `spent_txo_{count,sum}`)
-
- * `"U{scripthash}" → "{utxo}{blockhash}"` (where `utxo` is a set of `(txid,vout)` outpoints)
-
-#### Elements only:
-
-Stats for issued assets:
- * `"z{asset-id}" → "{issued_stats}{blockhash}"` (where `issued_stats` is composed of `tx_count`, `issuance_count`, `issued_amount`, `burned_amount`, `has_blinded_issuances`, `reissuance_tokens`, `burned_reissuance_tokens`)
-
-Stats for the native asset:
- * `"z{issued-asset}" → "{native_stats}{blockhash}"` (where `native_stats` is composed of `tx_count`, `peg_in_count`, `peg_in_amount`, `peg_out_count`, `peg_out_amount`, `burn_count` and `burn_amount`)
+Note that this mapping allows us to use `getrawtransaction` RPC to retrieve actual transaction data from without `-txindex` enabled
+(by explicitly specifying the [blockhash](https://github.com/bitcoin/bitcoin/commit/497d0e014cc79d46531d570e74e4aeae72db602d)).

doc/usage.md

@@ -1,6 +1,6 @@
 ## Installation
 
-Install [latest Rust](https://rustup.rs/) (1.31+),
+Install [latest Rust](https://rustup.rs/) (1.28+),
 [latest Bitcoin Core](https://bitcoincore.org/en/download/) (0.16+)
 and [latest Electrum wallet](https://electrum.org/#download) (3.2+).
 

docker-compose.yml

@@ -1,24 +0,0 @@
-version: '3.9'
-services:
-
-  mempool-electrs:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    restart: on-failure
-
-    ports:
-      - 50001:50001
-
-    entrypoint:
-      /bin/electrs
-    command: |
-      --address-search
-      --cookie mempool:mempool
-      --db-dir /electrs
-      --cors '*'
-    volumes:
-    - 'electrs_data:/electrs'
-
-volumes:
-  electrs_data:

init-electrs-sockets

@@ -1,24 +0,0 @@
-#!/usr/bin/env zsh
-export ZPOOL=nvm
-
-export BITCOIN_HOME=/bitcoin
-export BITCOIN_USER=bitcoin
-export BITCOIN_GROUP=bitcoin
-
-export ELEMENTS_HOME=/elements
-export ELEMENTS_USER=elements
-export ELEMENTS_GROUP=elements
-
-# create /bitcoin/socket with custom ACL for electrs unix sockets
-zfs create -o "mountpoint=${BITCOIN_HOME}/socket" "${ZPOOL}/bitcoin/socket"
-
-# create /elements/socket with custom ACL for electrs unix sockets
-zfs create -o "mountpoint=${ELEMENTS_HOME}/socket" "${ZPOOL}/elements/socket"
-
-setfacl -m "user:bitcoin:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${BITCOIN_HOME}/socket"
-
-chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket"
-
-setfacl -m "user:elements:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${ELEMENTS_HOME}/socket"
-
-chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket"

query/block.sh

@@ -0,0 +1,2 @@
+echo '{ "id": 5, "method":"blockchain.block.get", "params": ["997dd1addb13aac407fb7b996ca2f9cb4a9a71338d3ad9432bd62d2302939ac2"] }' | nc 127.0.0.1 51401
+

query/block_from_height.sh

@@ -0,0 +1,2 @@
+echo '{ "id": 5, "method":"blockchain.block.get_from_height", "params": [0] }' | nc 127.0.0.1 51401
+

query/hash_from_height.sh

@@ -0,0 +1,4 @@
+echo '{ "id": 5, "method":"blockchain.block.get_header", "params": [0] }' | nc 127.0.0.1 51401
+echo '{ "id": 5, "method":"blockchain.block.get_header", "params": [1] }' | nc 127.0.0.1 51401
+echo '{ "id": 5, "method":"blockchain.block.get_header", "params": [2] }' | nc 127.0.0.1 51401
+

query/height.sh

@@ -0,0 +1,2 @@
+echo '{ "id": 5, "method":"blockchain.headers.subscribe", "params": [] }' | nc 127.0.0.1 51401
+

query/version.sh

@@ -0,0 +1,2 @@
+echo "{ \"id\": 0, \"method\":\"server.version\", \"params\": [] }" | nc 127.0.0.1 51401
+

rust-toolchain

@@ -1 +0,0 @@
-1.87

scripts/checks.sh

@@ -1,64 +0,0 @@
-#!/bin/bash
-set -e
-
-# This script is used for running all the checks
-# needed to make sure CI passes.
-
-# See below for the reasoning behind testing
-# clippy for all feature combinations.
-# (Note: clippy reports compile errors too)
-
-# You can pass extra args to cargo commands
-#   ./scripts/check.sh --release
-# will run in release mode. You can use that
-# if you already have release mode compiled, as
-# it will be faster to use the version you already compiled
-
-TESTNAME=""
-cleanup() {
-    exit_code=$?
-    if [[ ${exit_code} -ne 0 ]]; then
-        echo -e "\n\n##### Failed on \"$TESTNAME\"" 1>&2
-    fi
-    exit $exit_code
-}
-trap cleanup EXIT
-
-TESTNAME="Running cargo check"
-echo "$TESTNAME"
-cargo check $@ -q --all-features
-
-TESTNAME="Running cargo fmt check"
-echo "$TESTNAME"
-cargo fmt $@ -q --all -- --check
-
-# Testing all the combinations of clippy.
-# There were many instances where a certain struct
-# differed based on liquid or not(liquid) etc.
-# and the clippy fixes would break the other
-# feature combination.
-#
-# Be prepared to use #[allow(clippy::___)] attributes
-# to "fix" contradictions between feature sets.
-
-TESTNAME="Running cargo clippy check no features"
-echo "$TESTNAME"
-cargo clippy $@ -q
-
-TESTNAME="Running cargo clippy check electrum-discovery"
-echo "$TESTNAME"
-cargo clippy $@ -q -F electrum-discovery
-
-TESTNAME="Running cargo clippy check liquid"
-echo "$TESTNAME"
-cargo clippy $@ -q -F liquid
-
-TESTNAME="Running cargo clippy check electrum-discovery + liquid"
-echo "$TESTNAME"
-cargo clippy $@ -q -F electrum-discovery,liquid
-
-if [ $INCLUDE_TESTS ]; then
-    TESTNAME="Running cargo test with all features"
-    echo "$TESTNAME"
-    cargo test $@ -q --lib --all-features
-fi

src/app.rs

@@ -1,9 +1,9 @@
-use bitcoin::hashes::sha256d::Hash as Sha256dHash;
+use bitcoin::util::hash::Sha256dHash;
 use std::sync::{Arc, Mutex};
 
-use crate::{daemon, index, signal::Waiter, store};
+use {daemon, index, signal::Waiter, store};
 
-use crate::errors::*;
+use errors::*;
 
 pub struct App {
     store: store::DBStore,

src/bin/electrs.rs

@@ -1,175 +1,90 @@
+extern crate electrs;
+
 extern crate error_chain;
 #[macro_use]
 extern crate log;
 
-extern crate electrs;
-
+use electrs::rest;
 use error_chain::ChainedError;
 use std::process;
-use std::sync::{Arc, RwLock};
 use std::time::Duration;
 
 use electrs::{
+    app::App,
+    bulk,
     config::Config,
     daemon::Daemon,
-    electrum::RPC as ElectrumRPC,
     errors::*,
+    index::Index,
     metrics::Metrics,
-    new_index::{precache, ChainQuery, FetchFrom, Indexer, Mempool, Query, Store},
-    rest,
+    query::Query,
     signal::Waiter,
+    store::{full_compaction, is_fully_compacted, verify_index_compatibility, DBStore},
 };
 
-#[cfg(feature = "liquid")]
-use electrs::elements::AssetRegistry;
-
-fn fetch_from(config: &Config, store: &Store) -> FetchFrom {
-    let mut jsonrpc_import = config.jsonrpc_import;
-    if !jsonrpc_import {
-        // switch over to jsonrpc after the initial sync is done
-        jsonrpc_import = store.done_initial_sync();
-    }
-
-    if jsonrpc_import {
-        // slower, uses JSONRPC (good for incremental updates)
-        FetchFrom::Bitcoind
-    } else {
-        // faster, uses blk*.dat files (good for initial indexing)
-        FetchFrom::BlkFiles
-    }
-}
-
-fn run_server(config: Arc<Config>) -> Result<()> {
-    let signal = Waiter::start();
+fn run_server(config: Config) -> Result<()> {
+    let signal = Waiter::new();
     let metrics = Metrics::new(config.monitoring_addr);
     metrics.start();
 
-    let daemon = Arc::new(Daemon::new(
-        config.daemon_dir.clone(),
-        config.blocks_dir.clone(),
+    let daemon = Daemon::new(
+        &config.daemon_dir,
         config.daemon_rpc_addr,
         config.cookie_getter(),
         config.network_type,
-        config.magic,
         signal.clone(),
         &metrics,
-    )?);
-    let store = Arc::new(Store::open(&config.db_path.join("newindex"), &config));
-    let mut indexer = Indexer::open(
-        Arc::clone(&store),
-        fetch_from(&config, &store),
-        &config,
-        &metrics,
-    );
-    let mut tip = indexer.update(&daemon)?;
+    )?;
+    // Perform initial indexing from local blk*.dat block files.
+    let store = DBStore::open(&config.db_path, /*low_memory=*/ config.jsonrpc_import);
+    let index = Index::load(&store, &daemon, &metrics, &config)?;
 
-    let chain = Arc::new(ChainQuery::new(
-        Arc::clone(&store),
-        Arc::clone(&daemon),
-        &config,
-        &metrics,
-    ));
+    verify_index_compatibility(&store, &config);
 
-    let mempool = Arc::new(RwLock::new(Mempool::new(
-        Arc::clone(&chain),
-        &metrics,
-        Arc::clone(&config),
-    )));
+    let store = if is_fully_compacted(&store) {
+        store // initial import and full compaction are over
+    } else {
+        if config.jsonrpc_import {
+            index.update(&store, &signal)?; // slower: uses JSONRPC for fetching blocks
+            full_compaction(store)
+        } else {
+            // faster, but uses more memory
+            let store = bulk::index_blk_files(&daemon, &config, &metrics, store)?;
+            let store = full_compaction(store);
+            index.reload(&store); // make sure the block header index is up-to-date
+            store
+        }
+    }
+    .enable_compaction(); // enable auto compactions before starting incremental index updates.
+
+    let app = App::new(store, index, daemon)?;
+    let query = Query::new(app.clone(), config.extended_db_enabled, &metrics);
+
+    let mut server = None; // HTTP REST server
     loop {
-        match Mempool::update(&mempool, &daemon) {
-            Ok(_) => break,
-            Err(e) => {
-                warn!(
-                    "Error performing initial mempool update, trying again in 5 seconds: {}",
-                    e.display_chain()
-                );
-                signal.wait(Duration::from_secs(5), false)?;
+        app.update(&signal)?;
+        query.update_mempool()?;
+
+        if server.is_none() {
+            if app.daemon().getblockchaininfo()?.verificationprogress > 0.9999 {
+                server = Some(rest::run_server(&config, query.clone()));
+            } else {
+                warn!("bitcoind not fully synced waiting");
             }
         }
-    }
 
-    #[cfg(feature = "liquid")]
-    let asset_db = config.asset_db_path.as_ref().map(|db_dir| {
-        let asset_db = Arc::new(RwLock::new(AssetRegistry::new(db_dir.clone())));
-        AssetRegistry::spawn_sync(asset_db.clone());
-        asset_db
-    });
-
-    let query = Arc::new(Query::new(
-        Arc::clone(&chain),
-        Arc::clone(&mempool),
-        Arc::clone(&daemon),
-        Arc::clone(&config),
-        #[cfg(feature = "liquid")]
-        asset_db,
-    ));
-
-    // TODO: configuration for which servers to start
-    let rest_server = rest::start(Arc::clone(&config), Arc::clone(&query), &metrics);
-    let electrum_server = ElectrumRPC::start(Arc::clone(&config), Arc::clone(&query), &metrics);
-
-    if let Some(ref precache_file) = config.precache_scripts {
-        let precache_scripthashes = precache::scripthashes_from_file(precache_file.to_string())
-            .expect("cannot load scripts to precache");
-        precache::precache(
-            Arc::clone(&chain),
-            precache_scripthashes,
-            config.precache_threads,
-        );
-    }
-
-    loop {
-        if let Err(err) = signal.wait(Duration::from_millis(config.main_loop_delay), true) {
+        if let Err(err) = signal.wait(Duration::from_secs(5)) {
             info!("stopping server: {}", err);
-
-            electrs::util::spawn_thread("shutdown-thread-checker", || {
-                let mut counter = 40;
-                let interval_ms = 500;
-
-                while counter > 0 {
-                    electrs::util::with_spawned_threads(|threads| {
-                        debug!("Threads during shutdown: {:?}", threads);
-                    });
-                    std::thread::sleep(std::time::Duration::from_millis(interval_ms));
-                    counter -= 1;
-                }
-            });
-
-            rest_server.stop();
-            // the electrum server is stopped when dropped
             break;
         }
-
-        // Index new blocks
-        let current_tip = daemon.getbestblockhash()?;
-        if current_tip != tip {
-            indexer.update(&daemon)?;
-            tip = current_tip;
-        };
-
-        // Update mempool
-        if let Err(e) = Mempool::update(&mempool, &daemon) {
-            // Log the error if the result is an Err
-            warn!(
-                "Error updating mempool, skipping mempool update: {}",
-                e.display_chain()
-            );
-        }
-
-        // Update subscribed clients
-        electrum_server.notify();
     }
-    info!("server stopped");
     Ok(())
 }
 
 fn main() {
-    let config = Arc::new(Config::from_args());
+    let config = Config::from_args();
     if let Err(e) = run_server(config) {
         error!("server failed: {}", e.display_chain());
         process::exit(1);
     }
-    electrs::util::with_spawned_threads(|threads| {
-        debug!("Threads before closing: {:?}", threads);
-    });
 }

src/bin/popular-scripts.rs

@@ -1,183 +0,0 @@
-extern crate electrs;
-
-use std::{convert::TryInto, thread::ThreadId, time::Instant};
-
-use electrs::{config::Config, new_index::db::open_raw_db};
-use lazy_static::lazy_static;
-
-/*
-// How to run:
-export ELECTRS_DATA=/path/to/electrs
-cargo run \
-  -q --release --bin popular-scripts -- \
-  --db-dir $ELECTRS_DATA/db \
-  > ./contrib/popular-scripts.txt
-*/
-
-type DB = rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>;
-lazy_static! {
-    static ref HISTORY_DB: DB = {
-        let config = Config::from_args();
-        open_raw_db(
-            &config.db_path.join("newindex").join("history"),
-            electrs::new_index::db::OpenMode::ReadOnly,
-        )
-    };
-}
-
-// Dev note:
-// Only use println for file output (lines for output)
-// Use eprintln to print to stderr for dev notifications
-fn main() {
-    let high_usage_threshold = std::env::var("HIGH_USAGE_THRESHOLD")
-        .ok()
-        .and_then(|s| s.parse::<u32>().ok())
-        .unwrap_or(4000);
-    let thread_count = std::env::var("JOB_THREAD_COUNT")
-        .ok()
-        .and_then(|s| s.parse::<usize>().ok())
-        .unwrap_or(4);
-    eprintln!(
-        "Seaching for scripts with history rows of {} or more...",
-        high_usage_threshold
-    );
-
-    let thread_pool = rayon::ThreadPoolBuilder::new()
-        .num_threads(thread_count)
-        .build()
-        .expect("Built threadpool");
-
-    let (sender, receiver) = crossbeam_channel::unbounded::<[u8; 32]>();
-
-    let increment = 256 / thread_count;
-    let bytes: Vec<u8> = (0u8..=255u8)
-        .filter(|n| *n % increment as u8 == 0)
-        .collect();
-
-    let now = Instant::now();
-    for i in 0..bytes.len() {
-        let sender = sender.clone();
-        let first_byte = bytes[i];
-        let second_byte = bytes.get(i + 1).copied();
-
-        thread_pool.spawn(move || {
-            let id = std::thread::current().id();
-            run_iterator(
-                id,
-                &HISTORY_DB,
-                high_usage_threshold,
-                first_byte,
-                second_byte,
-                sender,
-                now,
-            );
-            eprintln!("{id:?} Finished its job!");
-        })
-    }
-    // If we don't drop this sender
-    // the receiver will hang forever
-    drop(sender);
-
-    while let Ok(script) = receiver.recv() {
-        println!("{}", hex::encode(script));
-    }
-    eprintln!("Finished!!!!");
-}
-
-fn run_iterator(
-    thread_id: ThreadId,
-    db: &DB,
-    high_usage_threshold: u32,
-    first_byte: u8,
-    next_byte: Option<u8>,
-    sender: crossbeam_channel::Sender<[u8; 32]>,
-    now: Instant,
-) {
-    let mut iter = db.raw_iterator();
-    eprintln!(
-        "Thread ({thread_id:?}) Seeking DB to beginning of tx histories for b'H' + {}",
-        hex::encode([first_byte])
-    );
-    let mut compare_vec: Vec<u8> = vec![b'H', first_byte];
-    iter.seek(&compare_vec); // Seek to beginning of our section
-
-    // Insert the byte of the next section for comparing
-    // This will tell us when to stop with a closure
-    type Checker<'a> = Box<dyn Fn(&[u8]) -> bool + 'a>;
-    let is_finished: Checker<'_> = if let Some(next) = next_byte {
-        // Modify the vec to what we're looking for next
-        // to indicate we left our section
-        compare_vec[1] = next;
-        Box::new(|key: &[u8]| -> bool { key.starts_with(&compare_vec) })
-    } else {
-        // Modify the vec to only have H so we know when we left H
-        compare_vec.remove(1);
-        Box::new(|key: &[u8]| -> bool { !key.starts_with(&compare_vec) })
-    };
-
-    eprintln!("Thread ({thread_id:?}) Seeking done");
-
-    let mut curr_scripthash = [0u8; 32];
-    let mut total_entries: usize = 0;
-    let mut iter_index: usize = 1;
-
-    while iter.valid() {
-        let key = iter.key().unwrap();
-
-        if key.is_empty() || key[0] != b'H' || is_finished(key) {
-            // We have left the txhistory section,
-            // but we need to check the final scripthash
-            send_if_popular(
-                high_usage_threshold,
-                total_entries,
-                curr_scripthash,
-                &sender,
-            );
-            break;
-        }
-
-        if iter_index % 10_000_000 == 0 {
-            let duration = now.elapsed().as_secs();
-            eprintln!(
-                "Thread ({thread_id:?}) Processing row #{iter_index}... {duration} seconds elapsed"
-            );
-        }
-
-        // We know that the TxHistory key is 1 byte "H" followed by
-        // 32 byte scripthash
-        let entry_hash: [u8; 32] = key[1..33].try_into().unwrap();
-
-        if curr_scripthash != entry_hash {
-            // We have rolled on to a new scripthash
-            // If the last scripthash was popular
-            // Collect for sorting
-            send_if_popular(
-                high_usage_threshold,
-                total_entries,
-                curr_scripthash,
-                &sender,
-            );
-
-            // After collecting, reset values for next scripthash
-            curr_scripthash = entry_hash;
-            total_entries = 0;
-        }
-
-        total_entries += 1;
-        iter_index += 1;
-
-        iter.next();
-    }
-}
-
-#[inline]
-fn send_if_popular(
-    high_usage_threshold: u32,
-    total_entries: usize,
-    curr_scripthash: [u8; 32],
-    sender: &crossbeam_channel::Sender<[u8; 32]>,
-) {
-    if total_entries >= high_usage_threshold as usize {
-        sender.send(curr_scripthash).unwrap();
-    }
-}

src/bin/tx-fingerprint-stats.rs

@@ -1,160 +0,0 @@
-extern crate electrs;
-
-#[cfg(not(feature = "liquid"))]
-#[macro_use]
-extern crate log;
-
-#[cfg(not(feature = "liquid"))]
-fn main() {
-    use std::collections::HashSet;
-    use std::sync::Arc;
-
-    use bitcoin::blockdata::script::ScriptBuf;
-    use bitcoin::consensus::encode::deserialize;
-    use electrs::{
-        chain::Transaction,
-        config::Config,
-        daemon::Daemon,
-        metrics::Metrics,
-        new_index::{ChainQuery, FetchFrom, Indexer, Store},
-        signal::Waiter,
-        util::has_prevout,
-    };
-
-    let signal = Waiter::start();
-    let config = Config::from_args();
-    let store = Arc::new(Store::open(&config.db_path.join("newindex"), &config));
-
-    let metrics = Metrics::new(config.monitoring_addr);
-    metrics.start();
-
-    let daemon = Arc::new(
-        Daemon::new(
-            config.daemon_dir.clone(),
-            config.blocks_dir.clone(),
-            config.daemon_rpc_addr,
-            config.cookie_getter(),
-            config.network_type,
-            config.magic,
-            signal,
-            &metrics,
-        )
-        .unwrap(),
-    );
-
-    let chain = ChainQuery::new(Arc::clone(&store), Arc::clone(&daemon), &config, &metrics);
-
-    let mut indexer = Indexer::open(Arc::clone(&store), FetchFrom::Bitcoind, &config, &metrics);
-    indexer.update(&daemon).unwrap();
-
-    let mut iter = store.txstore_db().raw_iterator();
-    iter.seek(b"T");
-
-    let mut total = 0;
-    let mut uih_totals = vec![0, 0, 0];
-
-    while iter.valid() {
-        let key = iter.key().unwrap();
-        let value = iter.value().unwrap();
-
-        if !key.starts_with(b"T") {
-            break;
-        }
-
-        let tx: Transaction = deserialize(value).expect("failed to parse Transaction");
-        let txid = tx.compute_txid();
-
-        iter.next();
-
-        // only consider transactions of exactly two outputs
-        if tx.output.len() != 2 {
-            continue;
-        }
-        // skip coinbase txs
-        if tx.is_coinbase() {
-            continue;
-        }
-
-        // skip orphaned transactions
-        let blockid = match chain.tx_confirming_block(&txid) {
-            Some(blockid) => blockid,
-            None => continue,
-        };
-
-        //info!("{:?},{:?}", txid, blockid);
-
-        let prevouts = chain.lookup_txos(
-            &tx.input
-                .iter()
-                .filter(|txin| has_prevout(txin))
-                .map(|txin| txin.previous_output)
-                .collect(),
-        );
-
-        let total_out: u64 = tx.output.iter().map(|out| out.value.to_sat()).sum();
-        let small_out = tx
-            .output
-            .iter()
-            .map(|out| out.value.to_sat())
-            .min()
-            .unwrap();
-        let large_out = tx
-            .output
-            .iter()
-            .map(|out| out.value.to_sat())
-            .max()
-            .unwrap();
-
-        let total_in: u64 = prevouts.values().map(|out| out.value.to_sat()).sum();
-        let smallest_in = prevouts
-            .values()
-            .map(|out| out.value.to_sat())
-            .min()
-            .unwrap();
-
-        let fee = total_in - total_out;
-
-        // test for UIH
-        let uih = if total_in - smallest_in > large_out + fee {
-            2
-        } else if total_in - smallest_in > small_out + fee {
-            1
-        } else {
-            0
-        };
-
-        // test for spending multiple coins owned by the same spk
-        let is_multi_spend = {
-            let mut seen_spks = HashSet::new();
-            prevouts
-                .values()
-                .any(|out| !seen_spks.insert(&out.script_pubkey))
-        };
-
-        // test for sending back to one of the spent spks
-        let has_reuse = {
-            let prev_spks: HashSet<ScriptBuf> = prevouts
-                .values()
-                .map(|out| out.script_pubkey.clone())
-                .collect();
-            tx.output
-                .iter()
-                .any(|out| prev_spks.contains(&out.script_pubkey))
-        };
-
-        println!(
-            "{},{},{},{},{},{}",
-            txid, blockid.height, tx.lock_time, uih, is_multi_spend as u8, has_reuse as u8
-        );
-
-        total += 1;
-        uih_totals[uih] += 1;
-    }
-    info!(
-        "processed {} total txs, UIH counts: {:?}",
-        total, uih_totals
-    );
-}
-
-#[cfg(feature = "liquid")]
-fn main() {}

src/bulk.rs

@@ -0,0 +1,260 @@
+use bitcoin::consensus::encode::{deserialize, Decodable};
+use bitcoin::util::hash::{BitcoinHash, Sha256dHash};
+use elements::Block;
+use libc;
+use std::collections::HashSet;
+use std::fs;
+use std::io::{Cursor, Seek, SeekFrom};
+use std::path::{Path, PathBuf};
+use std::sync::{
+    mpsc::{Receiver, SyncSender},
+    Arc, Mutex,
+};
+use std::thread;
+
+use config::Config;
+use daemon::Daemon;
+use index::{index_block, last_indexed_block, read_indexed_blockhashes};
+use metrics::{CounterVec, Histogram, HistogramOpts, HistogramVec, MetricOpts, Metrics};
+use store::{DBStore, Row, WriteStore};
+use util::{spawn_thread, HeaderList, SyncChannel};
+
+use errors::*;
+
+struct Parser {
+    magic: u32,
+    current_headers: HeaderList,
+    indexed_blockhashes: Mutex<HashSet<Sha256dHash>>,
+    // metrics
+    duration: HistogramVec,
+    block_count: CounterVec,
+    bytes_read: Histogram,
+    extended_db_enabled: bool,
+}
+
+impl Parser {
+    fn new(
+        daemon: &Daemon,
+        metrics: &Metrics,
+        indexed_blockhashes: HashSet<Sha256dHash>,
+        extended_db_enabled: bool,
+    ) -> Result<Arc<Parser>> {
+        Ok(Arc::new(Parser {
+            magic: daemon.magic(),
+            current_headers: load_headers(daemon)?,
+            indexed_blockhashes: Mutex::new(indexed_blockhashes),
+            extended_db_enabled,
+            duration: metrics.histogram_vec(
+                HistogramOpts::new("parse_duration", "blk*.dat parsing duration (in seconds)"),
+                &["step"],
+            ),
+            block_count: metrics.counter_vec(
+                MetricOpts::new("parse_blocks", "# of block parsed (from blk*.dat)"),
+                &["type"],
+            ),
+
+            bytes_read: metrics.histogram(HistogramOpts::new(
+                "parse_bytes_read",
+                "# of bytes read (from blk*.dat)",
+            )),
+        }))
+    }
+
+    fn last_indexed_row(&self) -> Row {
+        let indexed_blockhashes = self.indexed_blockhashes.lock().unwrap();
+        let last_header = self
+            .current_headers
+            .iter()
+            .take_while(|h| indexed_blockhashes.contains(h.hash()))
+            .last()
+            .expect("no indexed header found");
+        debug!("last indexed block: {:?}", last_header);
+        last_indexed_block(last_header.hash())
+    }
+
+    fn read_blkfile(&self, path: &Path) -> Result<Vec<u8>> {
+        let timer = self.duration.with_label_values(&["read"]).start_timer();
+        let blob = fs::read(&path).chain_err(|| format!("failed to read {:?}", path))?;
+        timer.observe_duration();
+        self.bytes_read.observe(blob.len() as f64);
+        return Ok(blob);
+    }
+
+    fn index_blkfile(&self, blob: Vec<u8>) -> Result<Vec<Row>> {
+        let timer = self.duration.with_label_values(&["parse"]).start_timer();
+        let blocks = parse_blocks(blob, self.magic)?;
+        timer.observe_duration();
+
+        let mut rows = Vec::<Row>::new();
+        let timer = self.duration.with_label_values(&["index"]).start_timer();
+        for block in blocks {
+            let blockhash = block.bitcoin_hash();
+            if let Some(header) = self.current_headers.header_by_blockhash(&blockhash) {
+                if self
+                    .indexed_blockhashes
+                    .lock()
+                    .expect("indexed_blockhashes")
+                    .insert(blockhash.clone())
+                {
+                    rows.extend(index_block(
+                        &block,
+                        header.height() as u32,
+                        self.extended_db_enabled,
+                    ));
+                    self.block_count.with_label_values(&["indexed"]).inc();
+                } else {
+                    self.block_count.with_label_values(&["duplicate"]).inc();
+                }
+            } else {
+                // will be indexed later (after bulk load is over) if not an orphan block
+                self.block_count.with_label_values(&["skipped"]).inc();
+            }
+        }
+        timer.observe_duration();
+
+        let timer = self.duration.with_label_values(&["sort"]).start_timer();
+        rows.sort_unstable_by(|a, b| a.key.cmp(&b.key));
+        timer.observe_duration();
+        Ok(rows)
+    }
+}
+
+fn parse_blocks(blob: Vec<u8>, magic: u32) -> Result<Vec<Block>> {
+    let mut cursor = Cursor::new(&blob);
+    let mut blocks = vec![];
+    let max_pos = blob.len() as u64;
+    while cursor.position() < max_pos {
+        match u32::consensus_decode(&mut cursor) {
+            Ok(value) => {
+                if magic != value {
+                    cursor
+                        .seek(SeekFrom::Current(-3))
+                        .expect("failed to seek back");
+                    continue;
+                }
+            }
+            Err(_) => break, // EOF
+        };
+        let block_size = u32::consensus_decode(&mut cursor).chain_err(|| "no block size")?;
+        let start = cursor.position() as usize;
+        cursor
+            .seek(SeekFrom::Current(block_size as i64))
+            .chain_err(|| format!("seek {} failed", block_size))?;
+        let end = cursor.position() as usize;
+
+        let block: Block = deserialize(&blob[start..end])
+            .chain_err(|| format!("failed to parse block at {}..{}", start, end))?;
+        blocks.push(block);
+    }
+    Ok(blocks)
+}
+
+fn load_headers(daemon: &Daemon) -> Result<HeaderList> {
+    let tip = daemon.getbestblockhash()?;
+    let mut headers = HeaderList::empty();
+    let new_headers = headers.order(daemon.get_new_headers(&headers, &tip)?);
+    headers.apply(new_headers);
+    Ok(headers)
+}
+
+fn set_open_files_limit(limit: libc::rlim_t) {
+    let resource = libc::RLIMIT_NOFILE;
+    let mut rlim = libc::rlimit {
+        rlim_cur: 0,
+        rlim_max: 0,
+    };
+    let result = unsafe { libc::getrlimit(resource, &mut rlim) };
+    if result < 0 {
+        panic!("getrlimit() failed: {}", result);
+    }
+    rlim.rlim_cur = limit; // set softs limit only.
+    let result = unsafe { libc::setrlimit(resource, &rlim) };
+    if result < 0 {
+        panic!("setrlimit() failed: {}", result);
+    }
+}
+
+type JoinHandle = thread::JoinHandle<Result<()>>;
+type BlobReceiver = Arc<Mutex<Receiver<(Vec<u8>, PathBuf)>>>;
+
+fn start_reader(blk_files: Vec<PathBuf>, parser: Arc<Parser>) -> (BlobReceiver, JoinHandle) {
+    let chan = SyncChannel::new(0);
+    let blobs = chan.sender();
+    let handle = spawn_thread("bulk_read", move || -> Result<()> {
+        for path in blk_files {
+            blobs
+                .send((parser.read_blkfile(&path)?, path))
+                .expect("failed to send blk*.dat contents");
+        }
+        Ok(())
+    });
+    (Arc::new(Mutex::new(chan.into_receiver())), handle)
+}
+
+fn start_indexer(
+    blobs: BlobReceiver,
+    parser: Arc<Parser>,
+    writer: SyncSender<(Vec<Row>, PathBuf)>,
+) -> JoinHandle {
+    spawn_thread("bulk_index", move || -> Result<()> {
+        loop {
+            let msg = blobs.lock().unwrap().recv();
+            if let Ok((blob, path)) = msg {
+                let rows = parser
+                    .index_blkfile(blob)
+                    .chain_err(|| format!("failed to index {:?}", path))?;
+                writer
+                    .send((rows, path))
+                    .expect("failed to send indexed rows")
+            } else {
+                debug!("no more blocks to index");
+                break;
+            }
+        }
+        Ok(())
+    })
+}
+
+pub fn index_blk_files(
+    daemon: &Daemon,
+    config: &Config,
+    metrics: &Metrics,
+    store: DBStore,
+) -> Result<DBStore> {
+    set_open_files_limit(2048); // twice the default `ulimit -n` value
+    let blk_files = daemon.list_blk_files()?;
+    info!("indexing {} blk*.dat files", blk_files.len());
+    let indexed_blockhashes = read_indexed_blockhashes(&store);
+    debug!("found {} indexed blocks", indexed_blockhashes.len());
+    let parser = Parser::new(
+        daemon,
+        metrics,
+        indexed_blockhashes,
+        config.extended_db_enabled,
+    )?;
+    let (blobs, reader) = start_reader(blk_files, parser.clone());
+    let rows_chan = SyncChannel::new(0);
+    let indexers: Vec<JoinHandle> = (0..config.bulk_index_threads)
+        .map(|_| start_indexer(blobs.clone(), parser.clone(), rows_chan.sender()))
+        .collect();
+    Ok(spawn_thread("bulk_writer", move || -> DBStore {
+        for (rows, path) in rows_chan.into_receiver() {
+            trace!("indexed {:?}: {} rows", path, rows.len());
+            store.write(rows);
+        }
+        reader
+            .join()
+            .expect("reader panicked")
+            .expect("reader failed");
+
+        indexers.into_iter().for_each(|i| {
+            i.join()
+                .expect("indexer panicked")
+                .expect("indexing failed")
+        });
+        store.write(vec![parser.last_indexed_row()]);
+        store
+    })
+    .join()
+    .expect("writer panicked"))
+}

src/chain.rs

@@ -1,288 +0,0 @@
-use std::str::FromStr;
-
-#[cfg(not(feature = "liquid"))] // use regular Bitcoin data structures
-pub use bitcoin::{
-    address,
-    block::Header as BlockHeader,
-    blockdata::{opcodes, script},
-    consensus::deserialize,
-    hashes, Block, BlockHash, OutPoint, ScriptBuf as Script, Transaction, TxIn, TxOut, Txid,
-    Witness,
-};
-
-#[cfg(feature = "liquid")]
-pub use {
-    crate::elements::asset,
-    elements::{
-        address, bitcoin::bech32::Hrp, confidential, encode::deserialize, hashes, opcodes, script,
-        Address, AssetId, Block, BlockHash, BlockHeader, OutPoint, Script, Transaction, TxIn,
-        TxInWitness as Witness, TxOut, Txid,
-    },
-};
-
-use bitcoin::blockdata::constants::genesis_block;
-pub use bitcoin::Network as BNetwork;
-
-// Extension trait for getting txid in a cross-compatible way
-pub trait TxidCompat {
-    fn get_txid(&self) -> Txid;
-}
-
-#[cfg(not(feature = "liquid"))]
-impl TxidCompat for Transaction {
-    fn get_txid(&self) -> Txid {
-        self.compute_txid()
-    }
-}
-
-#[cfg(feature = "liquid")]
-impl TxidCompat for Transaction {
-    fn get_txid(&self) -> Txid {
-        self.txid()
-    }
-}
-
-// Extension trait for getting block size in a cross-compatible way
-pub trait BlockSizeCompat {
-    fn get_block_size(&self) -> usize;
-}
-
-#[cfg(not(feature = "liquid"))]
-impl BlockSizeCompat for Block {
-    fn get_block_size(&self) -> usize {
-        self.total_size()
-    }
-}
-
-#[cfg(feature = "liquid")]
-impl BlockSizeCompat for Block {
-    fn get_block_size(&self) -> usize {
-        self.size()
-    }
-}
-
-#[cfg(not(feature = "liquid"))]
-pub type Value = u64;
-#[cfg(feature = "liquid")]
-pub use confidential::Value;
-
-#[derive(Debug, Copy, Clone, PartialEq, Hash, Serialize, Ord, PartialOrd, Eq)]
-pub enum Network {
-    #[cfg(not(feature = "liquid"))]
-    Bitcoin,
-    #[cfg(not(feature = "liquid"))]
-    Testnet,
-    #[cfg(not(feature = "liquid"))]
-    Testnet4,
-    #[cfg(not(feature = "liquid"))]
-    Regtest,
-    #[cfg(not(feature = "liquid"))]
-    Signet,
-
-    #[cfg(feature = "liquid")]
-    Liquid,
-    #[cfg(feature = "liquid")]
-    LiquidTestnet,
-    #[cfg(feature = "liquid")]
-    LiquidRegtest,
-}
-
-#[cfg(feature = "liquid")]
-pub const LIQUID_TESTNET_PARAMS: address::AddressParams = address::AddressParams {
-    p2pkh_prefix: 36,
-    p2sh_prefix: 19,
-    blinded_prefix: 23,
-    bech_hrp: Hrp::parse_unchecked("tex"),
-    blech_hrp: Hrp::parse_unchecked("tlq"),
-};
-
-/// Magic for testnet4, 0x1c163f28 (from BIP94) with flipped endianness.
-#[cfg(not(feature = "liquid"))]
-const TESTNET4_MAGIC: u32 = 0x283f161c;
-
-impl Network {
-    #[cfg(not(feature = "liquid"))]
-    pub fn magic(self) -> u32 {
-        match self {
-            Self::Testnet4 => TESTNET4_MAGIC,
-            _ => {
-                let magic = BNetwork::from(self).magic();
-                u32::from_le_bytes(magic.to_bytes())
-            }
-        }
-    }
-
-    #[cfg(feature = "liquid")]
-    pub fn magic(self) -> u32 {
-        match self {
-            Network::Liquid | Network::LiquidRegtest => 0xDAB5_BFFA,
-            Network::LiquidTestnet => 0x62DD_0E41,
-        }
-    }
-
-    pub fn is_regtest(self) -> bool {
-        match self {
-            #[cfg(not(feature = "liquid"))]
-            Network::Regtest => true,
-            #[cfg(feature = "liquid")]
-            Network::LiquidRegtest => true,
-            _ => false,
-        }
-    }
-
-    #[cfg(feature = "liquid")]
-    pub fn address_params(self) -> &'static address::AddressParams {
-        // Liquid regtest uses elements's address params
-        match self {
-            Network::Liquid => &address::AddressParams::LIQUID,
-            Network::LiquidRegtest => &address::AddressParams::ELEMENTS,
-            Network::LiquidTestnet => &LIQUID_TESTNET_PARAMS,
-        }
-    }
-
-    #[cfg(feature = "liquid")]
-    pub fn native_asset(self) -> &'static AssetId {
-        match self {
-            Network::Liquid => &asset::NATIVE_ASSET_ID,
-            Network::LiquidTestnet => &asset::NATIVE_ASSET_ID_TESTNET,
-            Network::LiquidRegtest => &asset::NATIVE_ASSET_ID_REGTEST,
-        }
-    }
-
-    #[cfg(feature = "liquid")]
-    pub fn pegged_asset(self) -> Option<&'static AssetId> {
-        match self {
-            Network::Liquid => Some(&*asset::NATIVE_ASSET_ID),
-            Network::LiquidTestnet | Network::LiquidRegtest => None,
-        }
-    }
-
-    pub fn names() -> Vec<String> {
-        #[cfg(not(feature = "liquid"))]
-        return vec![
-            "mainnet".to_string(),
-            "testnet".to_string(),
-            "regtest".to_string(),
-            "signet".to_string(),
-        ];
-
-        #[cfg(feature = "liquid")]
-        return vec![
-            "liquid".to_string(),
-            "liquidtestnet".to_string(),
-            "liquidregtest".to_string(),
-        ];
-    }
-}
-
-pub fn genesis_hash(network: Network) -> BlockHash {
-    #[cfg(not(feature = "liquid"))]
-    return bitcoin_genesis_hash(network);
-    #[cfg(feature = "liquid")]
-    return liquid_genesis_hash(network);
-}
-
-pub fn bitcoin_genesis_hash(network: Network) -> bitcoin::BlockHash {
-    lazy_static! {
-        static ref BITCOIN_GENESIS: bitcoin::BlockHash =
-            genesis_block(BNetwork::Bitcoin).block_hash();
-        static ref TESTNET_GENESIS: bitcoin::BlockHash =
-            genesis_block(BNetwork::Testnet).block_hash();
-        static ref TESTNET4_GENESIS: bitcoin::BlockHash = bitcoin::BlockHash::from_str(
-            "00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043"
-        )
-        .unwrap();
-        static ref REGTEST_GENESIS: bitcoin::BlockHash =
-            genesis_block(BNetwork::Regtest).block_hash();
-        static ref SIGNET_GENESIS: bitcoin::BlockHash =
-            genesis_block(BNetwork::Signet).block_hash();
-    }
-    #[cfg(not(feature = "liquid"))]
-    match network {
-        Network::Bitcoin => *BITCOIN_GENESIS,
-        Network::Testnet => *TESTNET_GENESIS,
-        Network::Testnet4 => *TESTNET4_GENESIS,
-        Network::Regtest => *REGTEST_GENESIS,
-        Network::Signet => *SIGNET_GENESIS,
-    }
-    #[cfg(feature = "liquid")]
-    match network {
-        Network::Liquid => *BITCOIN_GENESIS,
-        Network::LiquidTestnet => *TESTNET_GENESIS,
-        Network::LiquidRegtest => *REGTEST_GENESIS,
-    }
-}
-
-#[cfg(feature = "liquid")]
-pub fn liquid_genesis_hash(network: Network) -> elements::BlockHash {
-    lazy_static! {
-        static ref LIQUID_GENESIS: BlockHash =
-            "1466275836220db2944ca059a3a10ef6fd2ea684b0688d2c379296888a206003"
-                .parse()
-                .unwrap();
-        static ref ZERO_HASH: BlockHash =
-            "0000000000000000000000000000000000000000000000000000000000000000"
-                .parse()
-                .unwrap();
-    }
-
-    match network {
-        Network::Liquid => *LIQUID_GENESIS,
-        // The genesis block for liquid regtest chains varies based on the chain configuration.
-        // This instead uses an all zeroed-out hash, which doesn't matter in practice because its
-        // only used for Electrum server discovery, which isn't active on regtest.
-        _ => *ZERO_HASH,
-    }
-}
-
-impl From<&str> for Network {
-    fn from(network_name: &str) -> Self {
-        match network_name {
-            #[cfg(not(feature = "liquid"))]
-            "mainnet" => Network::Bitcoin,
-            #[cfg(not(feature = "liquid"))]
-            "testnet" => Network::Testnet,
-            #[cfg(not(feature = "liquid"))]
-            "testnet4" => Network::Testnet4,
-            #[cfg(not(feature = "liquid"))]
-            "regtest" => Network::Regtest,
-            #[cfg(not(feature = "liquid"))]
-            "signet" => Network::Signet,
-
-            #[cfg(feature = "liquid")]
-            "liquid" => Network::Liquid,
-            #[cfg(feature = "liquid")]
-            "liquidtestnet" => Network::LiquidTestnet,
-            #[cfg(feature = "liquid")]
-            "liquidregtest" => Network::LiquidRegtest,
-
-            _ => panic!("unsupported Bitcoin network: {:?}", network_name),
-        }
-    }
-}
-
-#[cfg(not(feature = "liquid"))]
-impl From<Network> for BNetwork {
-    fn from(network: Network) -> Self {
-        match network {
-            Network::Bitcoin => BNetwork::Bitcoin,
-            Network::Testnet => BNetwork::Testnet,
-            Network::Testnet4 => BNetwork::Testnet4,
-            Network::Regtest => BNetwork::Regtest,
-            Network::Signet => BNetwork::Signet,
-        }
-    }
-}
-
-#[cfg(not(feature = "liquid"))]
-impl From<BNetwork> for Network {
-    fn from(network: BNetwork) -> Self {
-        match network {
-            BNetwork::Bitcoin => Network::Bitcoin,
-            BNetwork::Testnet => Network::Testnet,
-            BNetwork::Testnet4 => Network::Testnet4,
-            BNetwork::Regtest => Network::Regtest,
-            BNetwork::Signet => Network::Signet,
-        }
-    }
-}

src/config.rs

@@ -1,109 +1,43 @@
 use clap::{App, Arg};
 use dirs::home_dir;
+use num_cpus;
 use std::fs;
 use std::net::SocketAddr;
-use std::net::ToSocketAddrs;
 use std::path::{Path, PathBuf};
-use std::sync::{Arc, OnceLock};
+use std::sync::Arc;
 use stderrlog;
 
-use crate::chain::Network;
-use crate::daemon::CookieGetter;
-use crate::errors::*;
+use daemon::CookieGetter;
 
-#[cfg(feature = "liquid")]
-use bitcoin::Network as BNetwork;
-
-pub(crate) const APP_NAME: &str = "mempool-electrs";
-pub(crate) const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION");
-pub(crate) const GIT_HASH: Option<&str> = option_env!("GIT_HASH");
-// This will be set only once in the Daemon::new() constructor at startup
-pub(crate) static BITCOIND_SUBVER: OnceLock<String> = OnceLock::new();
-
-lazy_static! {
-    pub(crate) static ref VERSION_STRING: String = {
-        if let Some(hash) = GIT_HASH {
-            format!("{} {}-{}", APP_NAME, ELECTRS_VERSION, hash)
-        } else {
-            format!("{} {}", APP_NAME, ELECTRS_VERSION)
-        }
-    };
-}
+use daemon::Network;
+use errors::*;
 
 #[derive(Debug, Clone)]
 pub struct Config {
     // See below for the documentation of each field:
     pub log: stderrlog::StdErrLog,
     pub network_type: Network,
-    pub magic: Option<u32>,
+    pub parent_network: Network,
+    pub parent_genesis_hash: String,
     pub db_path: PathBuf,
     pub daemon_dir: PathBuf,
-    pub blocks_dir: PathBuf,
     pub daemon_rpc_addr: SocketAddr,
     pub cookie: Option<String>,
     pub electrum_rpc_addr: SocketAddr,
     pub http_addr: SocketAddr,
-    pub http_socket_file: Option<PathBuf>,
-    pub rpc_socket_file: Option<PathBuf>,
     pub monitoring_addr: SocketAddr,
     pub jsonrpc_import: bool,
-    pub light_mode: bool,
-    pub main_loop_delay: u64,
-    pub address_search: bool,
-    pub index_unspendables: bool,
-    pub cors: Option<String>,
-    pub precache_scripts: Option<String>,
-    pub precache_threads: usize,
-    pub utxos_limit: usize,
-    pub electrum_txs_limit: usize,
-    pub electrum_banner: String,
-    pub mempool_backlog_stats_ttl: u64,
-    pub mempool_recent_txs_size: usize,
-    pub rest_default_block_limit: usize,
-    pub rest_default_chain_txs_per_page: usize,
-    pub rest_default_max_mempool_txs: usize,
-    pub rest_default_max_address_summary_txs: usize,
-    pub rest_max_mempool_page_size: usize,
-    pub rest_max_mempool_txid_page_size: usize,
-    pub electrum_max_line_size: usize,
-    pub electrum_max_subscriptions: usize,
-    pub electrum_max_clients: usize,
-    pub electrum_idle_timeout: u64,
-    pub electrum_haproxy_depth: usize,
-    pub electrum_connections_per_client: usize,
-    pub electrum_public_hosts: Option<crate::electrum::ServerHosts>,
-
-    #[cfg(feature = "liquid")]
-    pub parent_network: BNetwork,
-    #[cfg(feature = "liquid")]
-    pub asset_db_path: Option<PathBuf>,
-
-    #[cfg(feature = "electrum-discovery")]
-    pub electrum_announce: bool,
-    #[cfg(feature = "electrum-discovery")]
-    pub tor_proxy: Option<std::net::SocketAddr>,
-}
-
-fn str_to_socketaddr(address: &str, what: &str) -> SocketAddr {
-    address
-        .to_socket_addrs()
-        .unwrap_or_else(|_| panic!("unable to resolve {} address", what))
-        .collect::<Vec<_>>()
-        .pop()
-        .unwrap()
+    pub index_batch_size: usize,
+    pub bulk_index_threads: usize,
+    pub tx_cache_size: usize,
+    pub extended_db_enabled: bool,
+    pub prevout_enabled: bool,
 }
 
 impl Config {
     pub fn from_args() -> Config {
-        let network_help = format!("Select network type ({})", Network::names().join(", "));
-
-        let args = App::new("Mempool Electrum Rust Server")
+        let m = App::new("Electrum Rust Server")
             .version(crate_version!())
-            .arg(
-                Arg::with_name("version")
-                    .long("version")
-                    .help("Print out the version of this app and quit immediately."),
-            )
             .arg(
                 Arg::with_name("verbosity")
                     .short("v")
@@ -127,12 +61,6 @@ impl Config {
                     .help("Data directory of Bitcoind (default: ~/.bitcoin/)")
                     .takes_value(true),
             )
-            .arg(
-                Arg::with_name("blocks_dir")
-                    .long("blocks-dir")
-                    .help("Analogous to bitcoind's -blocksdir option, this specifies the directory containing the raw blocks files (blk*.dat) (default: ~/.bitcoin/blocks/)")
-                    .takes_value(true),
-            )
             .arg(
                 Arg::with_name("cookie")
                     .long("cookie")
@@ -142,13 +70,13 @@ impl Config {
             .arg(
                 Arg::with_name("network")
                     .long("network")
-                    .help(&network_help)
+                    .help("Select Bitcoin network type ('mainnet', 'testnet', 'regtest', 'liquid', 'liquidregtest')")
                     .takes_value(true),
             )
             .arg(
-                Arg::with_name("magic")
-                    .long("magic")
-                    .default_value("")
+                Arg::with_name("parent_network")
+                    .long("parent-network")
+                    .help("Select parent network type ('mainnet', 'testnet', 'regtest', 'liquid', 'liquidregtest')")
                     .takes_value(true),
             )
             .arg(
@@ -160,7 +88,7 @@ impl Config {
             .arg(
                 Arg::with_name("http_addr")
                     .long("http-addr")
-                    .help("HTTP server 'addr:port' to listen on (default: '127.0.0.1:3000' for mainnet, '127.0.0.1:3001' for testnet and '127.0.0.1:3002' for regtest)")
+                    .help("HTTP server 'addr:port' to listen on (default: '127.0.0.1:3000' for mainnet/liquid, '127.0.0.1:3001' for testnet and '127.0.0.1:3002' for regtest/liquid-regtest)")
                     .takes_value(true),
             )
             .arg(
@@ -181,360 +109,112 @@ impl Config {
                     .help("Use JSONRPC instead of directly importing blk*.dat files. Useful for remote full node or low memory system"),
             )
             .arg(
-                Arg::with_name("light_mode")
-                    .long("lightmode")
-                    .help("Enable light mode for reduced storage")
+                Arg::with_name("index_batch_size")
+                    .long("index-batch-size")
+                    .help("Number of blocks to get in one JSONRPC request from bitcoind")
+                    .default_value("100"),
             )
             .arg(
-                Arg::with_name("main_loop_delay")
-                    .long("main-loop-delay")
-                    .help("The number of milliseconds the main loop will wait between loops. (Can be shortened with SIGUSR1)")
-                    .default_value("500")
-            )
-            .arg(
-                Arg::with_name("address_search")
-                    .long("address-search")
-                    .help("Enable prefix address search")
-            )
-            .arg(
-                Arg::with_name("index_unspendables")
-                    .long("index-unspendables")
-                    .help("Enable indexing of provably unspendable outputs")
-            )
-            .arg(
-                Arg::with_name("cors")
-                    .long("cors")
-                    .help("Origins allowed to make cross-site requests")
-                    .takes_value(true)
-            )
-            .arg(
-                Arg::with_name("precache_scripts")
-                    .long("precache-scripts")
-                    .help("Path to file with list of scripts to pre-cache")
-                    .takes_value(true)
-            )
-            .arg(
-                Arg::with_name("precache_threads")
-                    .long("precache-threads")
-                    .help("Non-zero number of threads to use for precache threadpool. [default: 4 * CORE_COUNT]")
-                    .takes_value(true)
-            )
-            .arg(
-                Arg::with_name("utxos_limit")
-                    .long("utxos-limit")
-                    .help("Maximum number of utxos to process per address. Lookups for addresses with more utxos will fail. Applies to the Electrum and HTTP APIs.")
-                    .default_value("500")
-            )
-            .arg(
-                Arg::with_name("mempool_backlog_stats_ttl")
-                    .long("mempool-backlog-stats-ttl")
-                    .help("The number of seconds that need to pass before Mempool::update will update the latency histogram again.")
-                    .default_value("10")
-            )
-            .arg(
-                Arg::with_name("mempool_recent_txs_size")
-                    .long("mempool-recent-txs-size")
-                    .help("The number of transactions that mempool will keep in its recents queue. This is returned by mempool/recent endpoint.")
-                    .default_value("10")
-            )
-            .arg(
-                Arg::with_name("rest_default_block_limit")
-                    .long("rest-default-block-limit")
-                    .help("The default number of blocks returned from the blocks/[start_height] endpoint.")
-                    .default_value("10")
-            )
-            .arg(
-                Arg::with_name("rest_default_chain_txs_per_page")
-                    .long("rest-default-chain-txs-per-page")
-                    .help("The default number of on-chain transactions returned by the txs endpoints.")
-                    .default_value("25")
-            )
-            .arg(
-                Arg::with_name("rest_default_max_mempool_txs")
-                    .long("rest-default-max-mempool-txs")
-                    .help("The default number of mempool transactions returned by the txs endpoints.")
-                    .default_value("50")
-            )
-            .arg(
-                Arg::with_name("rest_default_max_address_summary_txs")
-                    .long("rest-default-max-address-summary-txs")
-                    .help("The default number of transactions returned by the address summary endpoints.")
-                    .default_value("5000")
-            )
-            .arg(
-                Arg::with_name("rest_max_mempool_page_size")
-                    .long("rest-max-mempool-page-size")
-                    .help("The maximum number of transactions returned by the paginated /internal/mempool/txs endpoint.")
-                    .default_value("1000")
-            )
-            .arg(
-                Arg::with_name("rest_max_mempool_txid_page_size")
-                    .long("rest-max-mempool-txid-page-size")
-                    .help("The maximum number of transactions returned by the paginated /mempool/txids/page endpoint.")
-                    .default_value("10000")
-            )
-            .arg(
-                Arg::with_name("electrum_txs_limit")
-                    .long("electrum-txs-limit")
-                    .help("Maximum number of transactions returned by Electrum history queries. Lookups with more results will fail.")
-                    .default_value("500")
-            ).arg(
-                Arg::with_name("electrum_banner")
-                    .long("electrum-banner")
-                    .help("Welcome banner for the Electrum server, shown in the console to clients.")
-                    .takes_value(true)
-            ).arg(
-                Arg::with_name("electrum_max_line_size")
-                    .long("electrum-max-line-size")
-                    .help("Maximum size of a single Electrum request line in bytes (default: 1 MiB).")
-                    .default_value("1048576")
-            ).arg(
-                Arg::with_name("electrum_max_subscriptions")
-                    .long("electrum-max-subscriptions")
-                    .help("Maximum number of scripthash subscriptions per client connection.")
-                    .default_value("100")
-            ).arg(
-                Arg::with_name("electrum_max_clients")
-                    .long("electrum-max-clients")
-                    .help("Maximum number of concurrent Electrum client connections.")
-                    .default_value("10")
-            ).arg(
-                Arg::with_name("electrum_idle_timeout")
-                    .long("electrum-idle-timeout")
-                    .help("Maximum idle time in seconds since the last client request before disconnecting the Electrum connection.")
-                    .default_value("600")
-            ).arg(
-                Arg::with_name("electrum_haproxy_depth")
-                    .long("electrum-haproxy-depth")
-                    .help("Which HAProxy PROXY-protocol header layer identifies the real client IP. 0 disables PROXY-protocol detection; 1 uses the first (outermost) address, 2 the second, and so on. If the requested layer or any PROXY header is absent, no client IP is associated with the connection.")
+                Arg::with_name("bulk_index_threads")
+                    .long("bulk-index-threads")
+                    .help("Number of threads used for bulk indexing (default: use the # of CPUs)")
                     .default_value("0")
-            ).arg(
-                Arg::with_name("electrum_connections_per_client")
-                    .long("electrum-connections-per-client")
-                    .help("Maximum number of concurrent Electrum connections allowed per client (keyed by the HAProxy-reported address when available, otherwise the peer IP). 0 disables the per-client limit.")
-                    .default_value("10")
-            );
-
-        #[cfg(unix)]
-        let args = args.arg(
-                Arg::with_name("http_socket_file")
-                    .long("http-socket-file")
-                    .help("HTTP server 'unix socket file' to listen on (default disabled, enabling this disables the http server)")
-                    .takes_value(true),
-            );
-
-        #[cfg(unix)]
-        let args = args.arg(
-                Arg::with_name("rpc_socket_file")
-                    .long("rpc-socket-file")
-                    .help("Electrum RPC 'unix socket file' to listen on (default disabled, enabling this ignores the electrum_rpc_addr arg)")
-                    .takes_value(true),
-            );
-
-        #[cfg(feature = "liquid")]
-        let args = args
-            .arg(
-                Arg::with_name("parent_network")
-                    .long("parent-network")
-                    .help("Select parent network type (mainnet, testnet, regtest)")
-                    .takes_value(true),
             )
             .arg(
-                Arg::with_name("asset_db_path")
-                    .long("asset-db-path")
-                    .help("Directory for liquid/elements asset db")
-                    .takes_value(true),
-            );
-
-        #[cfg(feature = "electrum-discovery")]
-        let args = args.arg(
-                Arg::with_name("electrum_public_hosts")
-                    .long("electrum-public-hosts")
-                    .help("A dictionary of hosts where the Electrum server can be reached at. Required to enable server discovery. See https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server-features")
-                    .takes_value(true)
-            ).arg(
-                Arg::with_name("electrum_announce")
-                    .long("electrum-announce")
-                    .help("Announce the Electrum server to other servers")
-            ).arg(
-            Arg::with_name("tor_proxy")
-                .long("tor-proxy")
-                .help("ip:addr of socks proxy for accessing onion hosts")
-                .takes_value(true),
-        );
-
-        let m = args.get_matches();
-
-        if m.is_present("version") {
-            eprintln!("{}", *VERSION_STRING);
-            std::process::exit(0);
-        }
+                Arg::with_name("tx_cache_size")
+                    .long("tx-cache-size")
+                    .help("Number of transactions to keep in for query LRU cache")
+                    .default_value("10000")  // should be enough for a small wallet.
+            )
+            .arg(
+                Arg::with_name("light")
+                    .long("light")
+                    .help("Enable light operation mode")
+            )
+            .arg(
+                Arg::with_name("disable_prevout")
+                    .long("disable-prevout")
+                    .help("Don't attach previous output details to inputs")
+            )
+            .get_matches();
 
         let network_name = m.value_of("network").unwrap_or("mainnet");
         let network_type = Network::from(network_name);
-        let magic: Option<u32> = m
-            .value_of("magic")
-            .filter(|s| !s.is_empty())
-            .map(|s| u32::from_str_radix(s, 16).expect("invalid network magic"));
         let db_dir = Path::new(m.value_of("db_dir").unwrap_or("./db"));
         let db_path = db_dir.join(network_name);
 
-        #[cfg(feature = "liquid")]
-        let parent_network = m
-            .value_of("parent_network")
-            .map(|s| s.parse().expect("invalid parent network"))
-            .unwrap_or_else(|| match network_type {
-                Network::Liquid => BNetwork::Bitcoin,
-                // XXX liquid testnet/regtest don't have a parent chain
-                Network::LiquidTestnet | Network::LiquidRegtest => BNetwork::Regtest,
-            });
-
-        #[cfg(feature = "liquid")]
-        let asset_db_path = m.value_of("asset_db_path").map(PathBuf::from);
+        let parent_network = Network::from(m.value_of("parent_network").unwrap_or("mainnet"));
+        let parent_genesis_hash = parent_network.genesis_hash().le_hex_string();
 
         let default_daemon_port = match network_type {
-            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => 8332,
-            #[cfg(not(feature = "liquid"))]
             Network::Testnet => 18332,
-            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 18443,
-            #[cfg(not(feature = "liquid"))]
-            Network::Signet => 38332,
-            #[cfg(not(feature = "liquid"))]
-            Network::Testnet4 => 48332,
-
-            #[cfg(feature = "liquid")]
-            Network::Liquid => 7041,
-            #[cfg(feature = "liquid")]
-            Network::LiquidTestnet | Network::LiquidRegtest => 7040,
+            Network::Liquid => 10099,
+            Network::LiquidV1 => 7041,
+            Network::LiquidRegtest => 7041,
         };
         let default_electrum_port = match network_type {
-            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => 50001,
-            #[cfg(not(feature = "liquid"))]
             Network::Testnet => 60001,
-            #[cfg(not(feature = "liquid"))]
-            Network::Testnet4 => 40001,
-            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 60401,
-            #[cfg(not(feature = "liquid"))]
-            Network::Signet => 60601,
-
-            #[cfg(feature = "liquid")]
             Network::Liquid => 51000,
-            #[cfg(feature = "liquid")]
-            Network::LiquidTestnet => 51301,
-            #[cfg(feature = "liquid")]
+            Network::LiquidV1 => 51200,
             Network::LiquidRegtest => 51401,
         };
         let default_http_port = match network_type {
-            #[cfg(not(feature = "liquid"))]
-            Network::Bitcoin => 3000,
-            #[cfg(not(feature = "liquid"))]
+            Network::Bitcoin | Network::Liquid | Network::LiquidV1 => 3000,
             Network::Testnet => 3001,
-            #[cfg(not(feature = "liquid"))]
-            Network::Regtest => 3002,
-            #[cfg(not(feature = "liquid"))]
-            Network::Signet => 3003,
-            #[cfg(not(feature = "liquid"))]
-            Network::Testnet4 => 3004,
-
-            #[cfg(feature = "liquid")]
-            Network::Liquid => 3000,
-            #[cfg(feature = "liquid")]
-            Network::LiquidTestnet => 3001,
-            #[cfg(feature = "liquid")]
-            Network::LiquidRegtest => 3002,
+            Network::Regtest | Network::LiquidRegtest => 3002,
         };
         let default_monitoring_port = match network_type {
-            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => 4224,
-            #[cfg(not(feature = "liquid"))]
             Network::Testnet => 14224,
-            #[cfg(not(feature = "liquid"))]
             Network::Regtest => 24224,
-            #[cfg(not(feature = "liquid"))]
-            Network::Testnet4 => 44224,
-            #[cfg(not(feature = "liquid"))]
-            Network::Signet => 54224,
-
-            #[cfg(feature = "liquid")]
             Network::Liquid => 34224,
-            #[cfg(feature = "liquid")]
-            Network::LiquidTestnet => 44324,
-            #[cfg(feature = "liquid")]
+            Network::LiquidV1 => 31240,
             Network::LiquidRegtest => 44224,
         };
 
-        let daemon_rpc_addr: SocketAddr = str_to_socketaddr(
-            m.value_of("daemon_rpc_addr")
-                .unwrap_or(&format!("127.0.0.1:{}", default_daemon_port)),
-            "Bitcoin RPC",
-        );
-        let electrum_rpc_addr: SocketAddr = str_to_socketaddr(
-            m.value_of("electrum_rpc_addr")
-                .unwrap_or(&format!("127.0.0.1:{}", default_electrum_port)),
-            "Electrum RPC",
-        );
-        let http_addr: SocketAddr = str_to_socketaddr(
-            m.value_of("http_addr")
-                .unwrap_or(&format!("127.0.0.1:{}", default_http_port)),
-            "HTTP Server",
-        );
-
-        let http_socket_file: Option<PathBuf> = m.value_of("http_socket_file").map(PathBuf::from);
-        let rpc_socket_file: Option<PathBuf> = m.value_of("rpc_socket_file").map(PathBuf::from);
-        let monitoring_addr: SocketAddr = str_to_socketaddr(
-            m.value_of("monitoring_addr")
-                .unwrap_or(&format!("127.0.0.1:{}", default_monitoring_port)),
-            "Prometheus monitoring",
-        );
+        let daemon_rpc_addr: SocketAddr = m
+            .value_of("daemon_rpc_addr")
+            .unwrap_or(&format!("127.0.0.1:{}", default_daemon_port))
+            .parse()
+            .expect("invalid Bitcoind RPC address");
+        let electrum_rpc_addr: SocketAddr = m
+            .value_of("electrum_rpc_addr")
+            .unwrap_or(&format!("127.0.0.1:{}", default_electrum_port))
+            .parse()
+            .expect("invalid Electrum RPC address");
+        let http_addr: SocketAddr = m
+            .value_of("http_addr")
+            .unwrap_or(&format!("127.0.0.1:{}", default_http_port))
+            .parse()
+            .expect("invalid HTTP server address");
+        let monitoring_addr: SocketAddr = m
+            .value_of("monitoring_addr")
+            .unwrap_or(&format!("127.0.0.1:{}", default_monitoring_port))
+            .parse()
+            .expect("invalid Prometheus monitoring address");
 
         let mut daemon_dir = m
             .value_of("daemon_dir")
-            .map(PathBuf::from)
+            .map(|p| PathBuf::from(p))
             .unwrap_or_else(|| {
                 let mut default_dir = home_dir().expect("no homedir");
                 default_dir.push(".bitcoin");
                 default_dir
             });
         match network_type {
-            #[cfg(not(feature = "liquid"))]
             Network::Bitcoin => (),
-            #[cfg(not(feature = "liquid"))]
             Network::Testnet => daemon_dir.push("testnet3"),
-            #[cfg(not(feature = "liquid"))]
-            Network::Testnet4 => daemon_dir.push("testnet4"),
-            #[cfg(not(feature = "liquid"))]
             Network::Regtest => daemon_dir.push("regtest"),
-            #[cfg(not(feature = "liquid"))]
-            Network::Signet => daemon_dir.push("signet"),
-
-            #[cfg(feature = "liquid")]
-            Network::Liquid => daemon_dir.push("liquidv1"),
-            #[cfg(feature = "liquid")]
-            Network::LiquidTestnet => daemon_dir.push("liquidtestnet"),
-            #[cfg(feature = "liquid")]
+            Network::Liquid => daemon_dir.push("liquid"),
+            Network::LiquidV1 => daemon_dir.push("liquidv1"),
             Network::LiquidRegtest => daemon_dir.push("liquidregtest"),
         }
-        let blocks_dir = m
-            .value_of("blocks_dir")
-            .map(PathBuf::from)
-            .unwrap_or_else(|| daemon_dir.join("blocks"));
         let cookie = m.value_of("cookie").map(|s| s.to_owned());
 
-        let electrum_banner = m
-            .value_of("electrum_banner")
-            .map_or_else(|| format!("Welcome to {}", *VERSION_STRING), |s| s.into());
-
-        #[cfg(feature = "electrum-discovery")]
-        let electrum_public_hosts = m
-            .value_of("electrum_public_hosts")
-            .map(|s| serde_json::from_str(s).expect("invalid --electrum-public-hosts"));
-        #[cfg(not(feature = "electrum-discovery"))]
-        let electrum_public_hosts: Option<crate::electrum::ServerHosts> = None;
-
         let mut log = stderrlog::new();
         log.verbosity(m.occurrences_of("verbosity") as usize);
         log.timestamp(if m.is_present("timestamp") {
@@ -543,97 +223,34 @@ impl Config {
             stderrlog::Timestamp::Off
         });
         log.init().expect("logging initialization failed");
+        let mut bulk_index_threads = value_t_or_exit!(m, "bulk_index_threads", usize);
+        if bulk_index_threads == 0 {
+            bulk_index_threads = num_cpus::get();
+        }
         let config = Config {
             log,
             network_type,
-            magic,
+            parent_network,
+            parent_genesis_hash,
             db_path,
             daemon_dir,
-            blocks_dir,
             daemon_rpc_addr,
             cookie,
-            utxos_limit: value_t_or_exit!(m, "utxos_limit", usize),
             electrum_rpc_addr,
-            electrum_txs_limit: value_t_or_exit!(m, "electrum_txs_limit", usize),
-            electrum_banner,
             http_addr,
-            http_socket_file,
-            rpc_socket_file,
             monitoring_addr,
-            mempool_backlog_stats_ttl: value_t_or_exit!(m, "mempool_backlog_stats_ttl", u64),
-            mempool_recent_txs_size: value_t_or_exit!(m, "mempool_recent_txs_size", usize),
-            rest_default_block_limit: value_t_or_exit!(m, "rest_default_block_limit", usize),
-            rest_default_chain_txs_per_page: value_t_or_exit!(
-                m,
-                "rest_default_chain_txs_per_page",
-                usize
-            ),
-            rest_default_max_mempool_txs: value_t_or_exit!(
-                m,
-                "rest_default_max_mempool_txs",
-                usize
-            ),
-            rest_default_max_address_summary_txs: value_t_or_exit!(
-                m,
-                "rest_default_max_address_summary_txs",
-                usize
-            ),
-            rest_max_mempool_page_size: value_t_or_exit!(m, "rest_max_mempool_page_size", usize),
-            rest_max_mempool_txid_page_size: value_t_or_exit!(
-                m,
-                "rest_max_mempool_txid_page_size",
-                usize
-            ),
-            electrum_max_line_size: value_t_or_exit!(m, "electrum_max_line_size", usize),
-            electrum_max_subscriptions: value_t_or_exit!(m, "electrum_max_subscriptions", usize),
-            electrum_max_clients: value_t_or_exit!(m, "electrum_max_clients", usize),
-            electrum_idle_timeout: value_t_or_exit!(m, "electrum_idle_timeout", u64),
-            electrum_haproxy_depth: value_t_or_exit!(m, "electrum_haproxy_depth", usize),
-            electrum_connections_per_client: value_t_or_exit!(
-                m,
-                "electrum_connections_per_client",
-                usize
-            ),
             jsonrpc_import: m.is_present("jsonrpc_import"),
-            light_mode: m.is_present("light_mode"),
-            main_loop_delay: value_t_or_exit!(m, "main_loop_delay", u64),
-            address_search: m.is_present("address_search"),
-            index_unspendables: m.is_present("index_unspendables"),
-            cors: m.value_of("cors").map(|s| s.to_string()),
-            precache_scripts: m.value_of("precache_scripts").map(|s| s.to_string()),
-            precache_threads: m.value_of("precache_threads").map_or_else(
-                || {
-                    std::thread::available_parallelism()
-                        .expect("Can't get core count")
-                        .get()
-                        * 4
-                },
-                |s| match s.parse::<usize>() {
-                    Ok(v) if v > 0 => v,
-                    _ => clap::Error::value_validation_auto(format!(
-                        "The argument '{}' isn't a valid value",
-                        s
-                    ))
-                    .exit(),
-                },
-            ),
-
-            #[cfg(feature = "liquid")]
-            parent_network,
-            #[cfg(feature = "liquid")]
-            asset_db_path,
-
-            electrum_public_hosts,
-            #[cfg(feature = "electrum-discovery")]
-            electrum_announce: m.is_present("electrum_announce"),
-            #[cfg(feature = "electrum-discovery")]
-            tor_proxy: m.value_of("tor_proxy").map(|s| s.parse().unwrap()),
+            index_batch_size: value_t_or_exit!(m, "index_batch_size", usize),
+            bulk_index_threads,
+            tx_cache_size: value_t_or_exit!(m, "tx_cache_size", usize),
+            extended_db_enabled: !m.is_present("light"),
+            prevout_enabled: !m.is_present("disable_prevout"),
         };
         eprintln!("{:?}", config);
         config
     }
 
-    pub fn cookie_getter(&self) -> Arc<dyn CookieGetter> {
+    pub fn cookie_getter(&self) -> Arc<CookieGetter> {
         if let Some(ref value) = self.cookie {
             Arc::new(StaticCookie {
                 value: value.as_bytes().to_vec(),

src/daemon.rs

@@ -1,41 +1,101 @@
+use base64;
+use bitcoin::blockdata::constants::genesis_block;
+use bitcoin::consensus::encode::{deserialize, serialize};
+use bitcoin::network::constants::Network as BNetwork;
+use bitcoin::util::hash::BitcoinHash;
+use bitcoin::util::hash::Sha256dHash;
+use bitcoin_bech32::constants::Network as B32Network;
+use elements::{Block, BlockHeader, Transaction};
+use glob;
+use hex;
+use serde_json::{from_str, from_value, Value};
 use std::collections::{HashMap, HashSet};
 use std::io::{BufRead, BufReader, Lines, Write};
 use std::net::{SocketAddr, TcpStream};
 use std::path::PathBuf;
-use std::str::FromStr;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
-use base64;
-use bitcoin::hashes::Hash;
-use glob;
-use hex;
-use itertools::Itertools;
-use serde_json::{from_str, from_value, Value};
+use metrics::{HistogramOpts, HistogramVec, Metrics};
+use signal::Waiter;
+use util::HeaderList;
 
-#[cfg(not(feature = "liquid"))]
-use bitcoin::consensus::encode::{deserialize, serialize};
-#[cfg(feature = "liquid")]
-use elements::encode::{deserialize, serialize};
+use errors::*;
 
-use crate::chain::{Block, BlockHash, BlockHeader, Network, Transaction, Txid};
-use crate::config::BITCOIND_SUBVER;
-use crate::metrics::{HistogramOpts, HistogramVec, Metrics};
-use crate::signal::Waiter;
-use crate::util::HeaderList;
+#[derive(Debug, Copy, Clone, PartialEq, Hash, Serialize)]
+pub enum Network {
+    Bitcoin,
+    Testnet,
+    Regtest,
+    Liquid,
+    LiquidV1,
+    LiquidRegtest,
+}
 
-use crate::errors::*;
+impl Network {
+    pub fn genesis_hash(&self) -> Sha256dHash {
+        let block = genesis_block(BNetwork::from(self));
+        block.bitcoin_hash()
+    }
+}
 
-fn parse_hash<T>(value: &Value) -> Result<T>
-where
-    T: FromStr,
-    <T as FromStr>::Err: std::fmt::Debug,
-{
-    value
-        .as_str()
-        .chain_err(|| format!("non-string value: {}", value))?
-        .parse::<T>()
-        .map_err(|e| format!("failed to parse hash: {:?}", e).into())
+impl<'a> From<&'a str> for Network {
+    fn from(network_name: &'a str) -> Self {
+        match network_name {
+            "mainnet" => Network::Bitcoin,
+            "testnet" => Network::Testnet,
+            "regtest" => Network::Regtest,
+            "liquid" => Network::Liquid,
+            "liquidv1" => Network::LiquidV1,
+            "liquidregtest" => Network::LiquidRegtest,
+            _ => panic!("unsupported Bitcoin network: {:?}", network_name),
+        }
+    }
+}
+
+impl<'a> From<&'a Network> for BNetwork {
+    fn from(network: &'a Network) -> Self {
+        match network {
+            Network::Bitcoin => BNetwork::Bitcoin,
+            Network::Testnet => BNetwork::Testnet,
+            Network::Regtest => BNetwork::Regtest,
+            Network::Liquid => BNetwork::Bitcoin,   // @FIXME
+            Network::LiquidV1 => BNetwork::Bitcoin, // @FIXME
+            Network::LiquidRegtest => BNetwork::Regtest, // @FIXME
+        }
+    }
+}
+
+impl<'a> From<&'a Network> for B32Network {
+    fn from(network: &'a Network) -> Self {
+        match network {
+            Network::Bitcoin => B32Network::Bitcoin,
+            Network::Testnet => B32Network::Testnet,
+            Network::Regtest => B32Network::Regtest,
+            Network::Liquid => B32Network::Bitcoin, // @FIXME
+            Network::LiquidV1 => B32Network::Bitcoin, // @FIXME
+            Network::LiquidRegtest => B32Network::Regtest, // @FIXME
+        }
+    }
+}
+
+impl<'a> From<&'a BNetwork> for Network {
+    fn from(network: &'a BNetwork) -> Self {
+        match network {
+            BNetwork::Bitcoin => Network::Liquid,        // @FIXME
+            BNetwork::Regtest => Network::LiquidRegtest, // @FIXME
+            BNetwork::Testnet => Network::Testnet,       // @FIXME
+        }
+    }
+}
+
+fn parse_hash(value: &Value) -> Result<Sha256dHash> {
+    Ok(Sha256dHash::from_hex(
+        value
+            .as_str()
+            .chain_err(|| format!("non-string value: {}", value))?,
+    )
+    .chain_err(|| format!("non-hex value: {}", value))?)
 }
 
 fn header_from_value(value: Value) -> Result<BlockHeader> {
@@ -43,19 +103,22 @@ fn header_from_value(value: Value) -> Result<BlockHeader> {
         .as_str()
         .chain_err(|| format!("non-string header: {}", value))?;
     let header_bytes = hex::decode(header_hex).chain_err(|| "non-hex header")?;
-    deserialize(&header_bytes).chain_err(|| format!("failed to parse header {}", header_hex))
+    Ok(
+        deserialize(&header_bytes)
+            .chain_err(|| format!("failed to parse header {}", header_hex))?,
+    )
 }
 
 fn block_from_value(value: Value) -> Result<Block> {
     let block_hex = value.as_str().chain_err(|| "non-string block")?;
     let block_bytes = hex::decode(block_hex).chain_err(|| "non-hex block")?;
-    deserialize(&block_bytes).chain_err(|| format!("failed to parse block {}", block_hex))
+    Ok(deserialize(&block_bytes).chain_err(|| format!("failed to parse block {}", block_hex))?)
 }
 
 fn tx_from_value(value: Value) -> Result<Transaction> {
     let tx_hex = value.as_str().chain_err(|| "non-string tx")?;
     let tx_bytes = hex::decode(tx_hex).chain_err(|| "non-hex tx")?;
-    deserialize(&tx_bytes).chain_err(|| format!("failed to parse tx {}", tx_hex))
+    Ok(deserialize(&tx_bytes).chain_err(|| format!("failed to parse tx {}", tx_hex))?)
 }
 
 /// Parse JSONRPC error code, if exists.
@@ -67,13 +130,14 @@ fn parse_jsonrpc_reply(mut reply: Value, method: &str, expected_id: u64) -> Resu
     if let Some(reply_obj) = reply.as_object_mut() {
         if let Some(err) = reply_obj.get("error") {
             if !err.is_null() {
-                if let Some(code) = parse_error_code(err) {
+                if let Some(code) = parse_error_code(&err) {
                     match code {
                         // RPC_IN_WARMUP -> retry by later reconnection
                         -28 => bail!(ErrorKind::Connection(err.to_string())),
-                        _ => bail!("{} RPC error: {}", method, err),
+                        _ => (),
                     }
                 }
+                bail!("{} RPC error: {}", method, err);
             }
         }
         let id = reply_obj
@@ -104,67 +168,40 @@ pub struct BlockchainInfo {
     pub bestblockhash: String,
     pub pruned: bool,
     pub verificationprogress: f32,
-    pub initialblockdownload: Option<bool>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct MempoolInfo {
-    pub loaded: bool,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
 struct NetworkInfo {
     version: u64,
     subversion: String,
-    relayfee: f64, // in BTC/kB
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-struct MempoolFees {
-    base: f64,
-    #[serde(rename = "effective-feerate")]
-    effective_feerate: f64,
-    #[serde(rename = "effective-includes")]
-    effective_includes: Vec<String>,
+pub struct MempoolEntry {
+    fee: u64,   // in satoshis
+    vsize: u32, // in virtual bytes (= weight/4)
+    fee_per_vbyte: f32,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct MempoolAcceptResult {
-    txid: String,
-    wtxid: String,
-    allowed: Option<bool>,
-    vsize: Option<u32>,
-    fees: Option<MempoolFees>,
-    #[serde(rename = "reject-reason")]
-    reject_reason: Option<String>,
-}
+impl MempoolEntry {
+    fn new(fee: u64, vsize: u32) -> MempoolEntry {
+        MempoolEntry {
+            fee,
+            vsize,
+            fee_per_vbyte: fee as f32 / vsize as f32,
+        }
+    }
 
-#[derive(Serialize, Deserialize, Debug)]
-struct MempoolFeesSubmitPackage {
-    base: f64,
-    #[serde(rename = "effective-feerate")]
-    effective_feerate: Option<f64>,
-    #[serde(rename = "effective-includes")]
-    effective_includes: Option<Vec<String>>,
-}
+    pub fn fee_per_vbyte(&self) -> f32 {
+        self.fee_per_vbyte
+    }
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct SubmitPackageResult {
-    package_msg: String,
-    #[serde(rename = "tx-results")]
-    tx_results: HashMap<String, TxResult>,
-    #[serde(rename = "replaced-transactions")]
-    replaced_transactions: Option<Vec<String>>,
-}
+    pub fn fee(&self) -> u64 {
+        self.fee
+    }
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct TxResult {
-    txid: String,
-    #[serde(rename = "other-wtxid")]
-    other_wtxid: Option<String>,
-    vsize: Option<u32>,
-    fees: Option<MempoolFeesSubmitPackage>,
-    error: Option<String>,
+    pub fn vsize(&self) -> u32 {
+        self.vsize
+    }
 }
 
 pub trait CookieGetter: Send + Sync {
@@ -174,7 +211,7 @@ pub trait CookieGetter: Send + Sync {
 struct Connection {
     tx: TcpStream,
     rx: Lines<BufReader<TcpStream>>,
-    cookie_getter: Arc<dyn CookieGetter>,
+    cookie_getter: Arc<CookieGetter>,
     addr: SocketAddr,
     signal: Waiter,
 }
@@ -185,7 +222,7 @@ fn tcp_connect(addr: SocketAddr, signal: &Waiter) -> Result<TcpStream> {
             Ok(conn) => return Ok(conn),
             Err(err) => {
                 warn!("failed to connect daemon at {}: {}", addr, err);
-                signal.wait(Duration::from_secs(3), false)?;
+                signal.wait(Duration::from_secs(3))?;
                 continue;
             }
         }
@@ -195,7 +232,7 @@ fn tcp_connect(addr: SocketAddr, signal: &Waiter) -> Result<TcpStream> {
 impl Connection {
     fn new(
         addr: SocketAddr,
-        cookie_getter: Arc<dyn CookieGetter>,
+        cookie_getter: Arc<CookieGetter>,
         signal: Waiter,
     ) -> Result<Connection> {
         let conn = tcp_connect(addr, &signal)?;
@@ -239,7 +276,7 @@ impl Connection {
             .chain_err(|| {
                 ErrorKind::Connection("disconnected from daemon while receiving".to_owned())
             })?
-            .chain_err(|| ErrorKind::Connection("failed to read status".to_owned()))?;
+            .chain_err(|| "failed to read status")?;
         let mut headers = HashMap::new();
         for line in iter {
             let line = line.chain_err(|| ErrorKind::Connection("failed to read".to_owned()))?;
@@ -312,9 +349,7 @@ impl Counter {
 
 pub struct Daemon {
     daemon_dir: PathBuf,
-    blocks_dir: PathBuf,
     network: Network,
-    magic: Option<u32>,
     conn: Mutex<Connection>,
     message_id: Counter, // for monotonic JSONRPC 'id'
     signal: Waiter,
@@ -325,22 +360,17 @@ pub struct Daemon {
 }
 
 impl Daemon {
-    #[allow(clippy::too_many_arguments)]
     pub fn new(
-        daemon_dir: PathBuf,
-        blocks_dir: PathBuf,
+        daemon_dir: &PathBuf,
         daemon_rpc_addr: SocketAddr,
-        cookie_getter: Arc<dyn CookieGetter>,
+        cookie_getter: Arc<CookieGetter>,
         network: Network,
-        magic: Option<u32>,
         signal: Waiter,
         metrics: &Metrics,
     ) -> Result<Daemon> {
         let daemon = Daemon {
-            daemon_dir,
-            blocks_dir,
+            daemon_dir: daemon_dir.clone(),
             network,
-            magic,
             conn: Mutex::new(Connection::new(
                 daemon_rpc_addr,
                 cookie_getter,
@@ -359,52 +389,25 @@ impl Daemon {
         };
         let network_info = daemon.getnetworkinfo()?;
         info!("{:?}", network_info);
-        if network_info.version < 16_00_00 {
+        if network_info.version < 00_16_00_00 {
             bail!(
                 "{} is not supported - please use bitcoind 0.16+",
                 network_info.subversion,
             )
         }
-        // Insert the subversion (/Satoshi xx.xx.xx(comment)/) string from bitcoind
-        _ = BITCOIND_SUBVER.set(network_info.subversion);
-
         let blockchain_info = daemon.getblockchaininfo()?;
         info!("{:?}", blockchain_info);
-        if blockchain_info.pruned {
+        if blockchain_info.pruned == true {
             bail!("pruned node is not supported (use '-prune=0' bitcoind flag)".to_owned())
         }
-        loop {
-            let info = daemon.getblockchaininfo()?;
-            let mempool = daemon.getmempoolinfo()?;
 
-            let ibd_done = if network.is_regtest() {
-                info.blocks == info.headers
-            } else {
-                !info.initialblockdownload.unwrap_or(false)
-            };
-
-            if mempool.loaded && ibd_done && info.blocks == info.headers {
-                break;
-            }
-
-            warn!(
-                "waiting for bitcoind sync and mempool load to finish: {}/{} blocks, verification progress: {:.3}%, mempool loaded: {}",
-                info.blocks,
-                info.headers,
-                info.verificationprogress * 100.0,
-                mempool.loaded
-            );
-            signal.wait(Duration::from_secs(5), false)?;
-        }
         Ok(daemon)
     }
 
     pub fn reconnect(&self) -> Result<Daemon> {
         Ok(Daemon {
             daemon_dir: self.daemon_dir.clone(),
-            blocks_dir: self.blocks_dir.clone(),
             network: self.network,
-            magic: self.magic,
             conn: Mutex::new(self.conn.lock().unwrap().reconnect()?),
             message_id: Counter::new(),
             signal: self.signal.clone(),
@@ -414,8 +417,10 @@ impl Daemon {
     }
 
     pub fn list_blk_files(&self) -> Result<Vec<PathBuf>> {
-        let path = self.blocks_dir.join("blk*.dat");
-        debug!("listing block files at {:?}", path);
+        let mut path = self.daemon_dir.clone();
+        path.push("blocks");
+        path.push("blk*.dat");
+        info!("listing block files at {:?}", path);
         let mut paths: Vec<PathBuf> = glob::glob(path.to_str().unwrap())
             .chain_err(|| "failed to list blk*.dat files")?
             .map(|res| res.unwrap())
@@ -425,7 +430,14 @@ impl Daemon {
     }
 
     pub fn magic(&self) -> u32 {
-        self.magic.unwrap_or_else(|| self.network.magic())
+        match self.network {
+            Network::Bitcoin => 0xD9B4BEF9,
+            Network::Testnet => 0x0709110B,
+            Network::Regtest => 0xDAB5BFFA,
+            Network::Liquid => 0xDAB5BFFA,
+            Network::LiquidV1 => 0xDAB5BFFA,
+            Network::LiquidRegtest => 0xDAB5BFFA,
+        }
     }
 
     fn call_jsonrpc(&self, method: &str, request: &Value) -> Result<Value> {
@@ -445,66 +457,29 @@ impl Daemon {
         Ok(result)
     }
 
-    fn handle_request_batch(
-        &self,
-        method: &str,
-        params_list: &[Value],
-        failure_threshold: f64,
-    ) -> Result<Vec<Value>> {
+    fn handle_request_batch(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
         let id = self.message_id.next();
-        let chunks = params_list
+        let reqs = params_list
             .iter()
             .map(|params| json!({"method": method, "params": params, "id": id}))
-            .chunks(50_000); // Max Amount of batched requests
+            .collect();
         let mut results = vec![];
-        let total_requests = params_list.len();
-        let mut failed_requests: u64 = 0;
-        let threshold = (failure_threshold * total_requests as f64).round() as u64;
-        let mut n = 0;
-
-        for chunk in &chunks {
-            let reqs = chunk.collect();
-            let mut replies = self.call_jsonrpc(method, &reqs)?;
-            if let Some(replies_vec) = replies.as_array_mut() {
-                for reply in replies_vec {
-                    n += 1;
-                    match parse_jsonrpc_reply(reply.take(), method, id) {
-                        Ok(parsed_reply) => results.push(parsed_reply),
-                        Err(e) => {
-                            failed_requests += 1;
-                            warn!(
-                                "batch request {} {}/{} failed: {}",
-                                method,
-                                n,
-                                total_requests,
-                                e.to_string()
-                            );
-                            // abort and return the last error once a threshold number of requests have failed
-                            if failed_requests > threshold {
-                                return Err(e);
-                            }
-                        }
-                    }
-                }
-            } else {
-                bail!("non-array replies: {:?}", replies);
+        let mut replies = self.call_jsonrpc(method, &reqs)?;
+        if let Some(replies_vec) = replies.as_array_mut() {
+            for reply in replies_vec {
+                results.push(parse_jsonrpc_reply(reply.take(), method, id)?)
             }
+            return Ok(results);
         }
-
-        Ok(results)
+        bail!("non-array replies: {:?}", replies);
     }
 
-    fn retry_request_batch(
-        &self,
-        method: &str,
-        params_list: &[Value],
-        failure_threshold: f64,
-    ) -> Result<Vec<Value>> {
+    fn retry_request_batch(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
         loop {
-            match self.handle_request_batch(method, params_list, failure_threshold) {
+            match self.handle_request_batch(method, params_list) {
                 Err(Error(ErrorKind::Connection(msg), _)) => {
                     warn!("reconnecting to bitcoind: {}", msg);
-                    self.signal.wait(Duration::from_secs(3), false)?;
+                    self.signal.wait(Duration::from_secs(3))?;
                     let mut conn = self.conn.lock().unwrap();
                     *conn = conn.reconnect()?;
                     continue;
@@ -515,40 +490,35 @@ impl Daemon {
     }
 
     fn request(&self, method: &str, params: Value) -> Result<Value> {
-        let mut values = self.retry_request_batch(method, &[params], 0.0)?;
+        let mut values = self.retry_request_batch(method, &[params])?;
         assert_eq!(values.len(), 1);
         Ok(values.remove(0))
     }
 
     fn requests(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
-        self.retry_request_batch(method, params_list, 0.0)
+        self.retry_request_batch(method, params_list)
     }
 
     // bitcoind JSONRPC API:
 
     pub fn getblockchaininfo(&self) -> Result<BlockchainInfo> {
         let info: Value = self.request("getblockchaininfo", json!([]))?;
-        from_value(info).chain_err(|| "invalid blockchain info")
-    }
-
-    fn getmempoolinfo(&self) -> Result<MempoolInfo> {
-        let info: Value = self.request("getmempoolinfo", json!([]))?;
-        from_value(info).chain_err(|| "invalid mempool info")
+        Ok(from_value(info).chain_err(|| "invalid blockchain info")?)
     }
 
     fn getnetworkinfo(&self) -> Result<NetworkInfo> {
         let info: Value = self.request("getnetworkinfo", json!([]))?;
-        from_value(info).chain_err(|| "invalid network info")
+        Ok(from_value(info).chain_err(|| "invalid network info")?)
     }
 
-    pub fn getbestblockhash(&self) -> Result<BlockHash> {
-        parse_hash(&self.request("getbestblockhash", json!([]))?)
+    pub fn getbestblockhash(&self) -> Result<Sha256dHash> {
+        parse_hash(&self.request("getbestblockhash", json!([]))?).chain_err(|| "invalid blockhash")
     }
 
-    pub fn getblockheader(&self, blockhash: &BlockHash) -> Result<BlockHeader> {
+    pub fn getblockheader(&self, blockhash: &Sha256dHash) -> Result<BlockHeader> {
         header_from_value(self.request(
             "getblockheader",
-            json!([blockhash.to_string(), /*verbose=*/ false]),
+            json!([blockhash.be_hex_string(), /*verbose=*/ false]),
         )?)
     }
 
@@ -566,23 +536,23 @@ impl Daemon {
         Ok(result)
     }
 
-    pub fn getblock(&self, blockhash: &BlockHash) -> Result<Block> {
+    pub fn getblock(&self, blockhash: &Sha256dHash) -> Result<Block> {
         let block = block_from_value(self.request(
             "getblock",
-            json!([blockhash.to_string(), /*verbose=*/ false]),
+            json!([blockhash.be_hex_string(), /*verbose=*/ false]),
         )?)?;
-        assert_eq!(block.block_hash(), *blockhash);
+        assert_eq!(block.bitcoin_hash(), *blockhash);
         Ok(block)
     }
 
-    pub fn getblock_raw(&self, blockhash: &BlockHash, verbose: u32) -> Result<Value> {
-        self.request("getblock", json!([blockhash.to_string(), verbose]))
+    pub fn getblock_raw(&self, blockhash: &Sha256dHash, verbose: u32) -> Result<Value> {
+        self.request("getblock", json!([blockhash.be_hex_string(), verbose]))
     }
 
-    pub fn getblocks(&self, blockhashes: &[BlockHash]) -> Result<Vec<Block>> {
+    pub fn getblocks(&self, blockhashes: &[Sha256dHash]) -> Result<Vec<Block>> {
         let params_list: Vec<Value> = blockhashes
             .iter()
-            .map(|hash| json!([hash.to_string(), /*verbose=*/ false]))
+            .map(|hash| json!([hash.be_hex_string(), /*verbose=*/ false]))
             .collect();
         let values = self.requests("getblock", &params_list)?;
         let mut blocks = vec![];
@@ -592,145 +562,88 @@ impl Daemon {
         Ok(blocks)
     }
 
-    pub fn gettransactions(&self, txhashes: &[&Txid]) -> Result<Vec<Transaction>> {
+    pub fn gettransaction(&self, txhash: &Sha256dHash) -> Result<Transaction> {
+        let args = json!([txhash.be_hex_string(), /*verbose=*/ false]);
+        tx_from_value(self.request("getrawtransaction", args)?)
+    }
+
+    pub fn gettransaction_raw(&self, txhash: &Sha256dHash, verbose: bool) -> Result<Value> {
+        let args = json!([txhash.be_hex_string(), verbose]);
+        debug!("gettransaction_raw args {:?}", args);
+        Ok(self.request("getrawtransaction", args)?)
+    }
+
+    pub fn gettransactions(&self, txhashes: &[&Sha256dHash]) -> Result<Vec<Transaction>> {
         let params_list: Vec<Value> = txhashes
             .iter()
-            .map(|txhash| json!([txhash.to_string(), /*verbose=*/ false]))
+            .map(|txhash| json!([txhash.be_hex_string(), /*verbose=*/ false]))
             .collect();
-        let values = self.retry_request_batch("getrawtransaction", &params_list, 0.25)?;
+
+        let values = self.requests("getrawtransaction", &params_list)?;
         let mut txs = vec![];
         for value in values {
             txs.push(tx_from_value(value)?);
         }
-        // missing transactions are skipped, so the number of txs returned may be less than the number of txids requested
+        assert_eq!(txhashes.len(), txs.len());
         Ok(txs)
     }
 
-    pub fn gettransaction_raw(
-        &self,
-        txid: &Txid,
-        blockhash: &BlockHash,
-        verbose: bool,
-    ) -> Result<Value> {
-        self.request(
-            "getrawtransaction",
-            json!([txid.to_string(), verbose, blockhash]),
+    pub fn getmempooltxids(&self) -> Result<HashSet<Sha256dHash>> {
+        let txids: Value = self.request("getrawmempool", json!([/*verbose=*/ false]))?;
+        let mut result = HashSet::new();
+        for value in txids.as_array().chain_err(|| "non-array result")? {
+            result.insert(parse_hash(&value).chain_err(|| "invalid txid")?);
+        }
+        Ok(result)
+    }
+
+    pub fn getmempoolentry(&self, txid: &Sha256dHash) -> Result<MempoolEntry> {
+        let entry = self.request("getmempoolentry", json!([txid.be_hex_string()]))?;
+        let fee = (entry
+            .get("fee")
+            .chain_err(|| "missing fee")?
+            .as_f64()
+            .chain_err(|| "non-float fee")?
+            * 100_000_000f64) as u64;
+        let vsize = entry
+            .get("size")
+            .chain_err(|| "missing size")?
+            .as_u64()
+            .chain_err(|| "non-integer size")? as u32;
+        Ok(MempoolEntry::new(fee, vsize))
+    }
+
+    pub fn broadcast(&self, tx: &Transaction) -> Result<Sha256dHash> {
+        let tx = hex::encode(serialize(tx));
+        let txid = self.request("sendrawtransaction", json!([tx]))?;
+        Ok(
+            Sha256dHash::from_hex(txid.as_str().chain_err(|| "non-string txid")?)
+                .chain_err(|| "failed to parse txid")?,
         )
     }
 
-    pub fn getmempooltx(&self, txhash: &Txid) -> Result<Transaction> {
-        let value = self.request(
-            "getrawtransaction",
-            json!([txhash.to_string(), /*verbose=*/ false]),
-        )?;
-        tx_from_value(value)
-    }
-
-    pub fn getmempooltxids(&self) -> Result<HashSet<Txid>> {
-        let res = self.request("getrawmempool", json!([/*verbose=*/ false]))?;
-        serde_json::from_value(res).chain_err(|| "invalid getrawmempool reply")
-    }
-
-    pub fn broadcast(&self, tx: &Transaction) -> Result<Txid> {
-        self.broadcast_raw(&hex::encode(serialize(tx)))
-    }
-
-    pub fn broadcast_raw(&self, txhex: &str) -> Result<Txid> {
-        let txid = self.request("sendrawtransaction", json!([txhex]))?;
-        txid.as_str()
-            .chain_err(|| "non-string txid")?
-            .parse::<Txid>()
-            .map_err(|e| format!("failed to parse txid: {:?}", e).into())
-    }
-
-    pub fn test_mempool_accept(
-        &self,
-        txhex: Vec<String>,
-        maxfeerate: Option<f64>,
-    ) -> Result<Vec<MempoolAcceptResult>> {
-        let params = match maxfeerate {
-            Some(rate) => json!([txhex, format!("{:.8}", rate)]),
-            None => json!([txhex]),
-        };
-        let result = self.request("testmempoolaccept", params)?;
-        serde_json::from_value::<Vec<MempoolAcceptResult>>(result)
-            .chain_err(|| "invalid testmempoolaccept reply")
-    }
-
-    pub fn submit_package(
-        &self,
-        txhex: Vec<String>,
-        maxfeerate: Option<f64>,
-        maxburnamount: Option<f64>,
-    ) -> Result<SubmitPackageResult> {
-        let params = match (maxfeerate, maxburnamount) {
-            (Some(rate), Some(burn)) => {
-                json!([txhex, format!("{:.8}", rate), format!("{:.8}", burn)])
-            }
-            (Some(rate), None) => json!([txhex, format!("{:.8}", rate)]),
-            (None, Some(burn)) => json!([txhex, null, format!("{:.8}", burn)]),
-            (None, None) => json!([txhex]),
-        };
-        let result = self.request("submitpackage", params)?;
-        serde_json::from_value::<SubmitPackageResult>(result)
-            .chain_err(|| "invalid submitpackage reply")
-    }
-
-    // Get estimated feerates for the provided confirmation targets using a batch RPC request
-    // Missing estimates are logged but do not cause a failure, whatever is available is returned
-    #[allow(clippy::float_cmp)]
-    pub fn estimatesmartfee_batch(&self, conf_targets: &[u16]) -> Result<HashMap<u16, f64>> {
-        let params_list: Vec<Value> = conf_targets.iter().map(|t| json!([t])).collect();
-
-        Ok(self
-            .requests("estimatesmartfee", &params_list)?
-            .iter()
-            .zip(conf_targets)
-            .filter_map(|(reply, target)| {
-                if !reply["errors"].is_null() {
-                    warn!(
-                        "failed estimating fee for target {}: {:?}",
-                        target, reply["errors"]
-                    );
-                    return None;
-                }
-
-                let feerate = reply["feerate"]
-                    .as_f64()
-                    .unwrap_or_else(|| panic!("invalid estimatesmartfee response: {:?}", reply));
-
-                if feerate == -1f64 {
-                    warn!("not enough data to estimate fee for target {}", target);
-                    return None;
-                }
-
-                // from BTC/kB to sat/b
-                Some((*target, feerate * 100_000f64))
-            })
-            .collect())
-    }
-
-    fn get_all_headers(&self, tip: &BlockHash) -> Result<Vec<BlockHeader>> {
-        let info: Value = self.request("getblockheader", json!([tip.to_string()]))?;
+    fn get_all_headers(&self, tip: &Sha256dHash) -> Result<Vec<BlockHeader>> {
+        let info: Value = self.request("getblockheader", json!([tip.be_hex_string()]))?;
         let tip_height = info
             .get("height")
             .expect("missing height")
             .as_u64()
             .expect("non-numeric height") as usize;
-        let all_heights: Vec<usize> = (0..=tip_height).collect();
+        let all_heights: Vec<usize> = (0..tip_height + 1).collect();
         let chunk_size = 100_000;
         let mut result = vec![];
+        let null_hash = Sha256dHash::default();
         for heights in all_heights.chunks(chunk_size) {
             trace!("downloading {} block headers", heights.len());
-            let mut headers = self.getblockheaders(heights)?;
+            let mut headers = self.getblockheaders(&heights)?;
             assert!(headers.len() == heights.len());
             result.append(&mut headers);
         }
 
-        let mut blockhash = BlockHash::all_zeros();
+        let mut blockhash = null_hash;
         for header in &result {
             assert_eq!(header.prev_blockhash, blockhash);
-            blockhash = header.block_hash();
+            blockhash = header.bitcoin_hash();
         }
         assert_eq!(blockhash, *tip);
         Ok(result)
@@ -740,11 +653,10 @@ impl Daemon {
     pub fn get_new_headers(
         &self,
         indexed_headers: &HeaderList,
-        bestblockhash: &BlockHash,
+        bestblockhash: &Sha256dHash,
     ) -> Result<Vec<BlockHeader>> {
         // Iterate back over headers until known blockash is found:
-        if indexed_headers.is_empty() {
-            debug!("downloading all block headers up to {}", bestblockhash);
+        if indexed_headers.len() == 0 {
             return self.get_all_headers(bestblockhash);
         }
         debug!(
@@ -753,7 +665,7 @@ impl Daemon {
             bestblockhash,
         );
         let mut new_headers = vec![];
-        let null_hash = BlockHash::all_zeros();
+        let null_hash = Sha256dHash::default();
         let mut blockhash = *bestblockhash;
         while blockhash != null_hash {
             if indexed_headers.header_by_blockhash(&blockhash).is_some() {
@@ -762,18 +674,11 @@ impl Daemon {
             let header = self
                 .getblockheader(&blockhash)
                 .chain_err(|| format!("failed to get {} header", blockhash))?;
+            new_headers.push(header.clone());
             blockhash = header.prev_blockhash;
-            new_headers.push(header);
         }
         trace!("downloaded {} block headers", new_headers.len());
         new_headers.reverse(); // so the tip is the last vector entry
         Ok(new_headers)
     }
-
-    pub fn get_relayfee(&self) -> Result<f64> {
-        let relayfee = self.getnetworkinfo()?.relayfee;
-
-        // from BTC/kB to sat/b
-        Ok(relayfee * 100_000f64)
-    }
 }

src/electrum/client.rs

@@ -1,41 +0,0 @@
-use std::collections::HashMap;
-use std::convert::TryFrom;
-
-use bitcoin::hashes::sha256d;
-use bitcoin::hashes::Hash;
-pub use electrum_client::client::Client;
-pub use electrum_client::ServerFeaturesRes;
-
-use crate::chain::BlockHash;
-use crate::electrum::ServerFeatures;
-use crate::errors::{Error, ResultExt};
-
-// Convert from electrum-client's server features struct to ours. We're using a different struct because
-// the electrum-client's one doesn't support the "hosts" key.
-impl TryFrom<ServerFeaturesRes> for ServerFeatures {
-    type Error = Error;
-    fn try_from(mut features: ServerFeaturesRes) -> Result<Self, Self::Error> {
-        features.genesis_hash.reverse();
-
-        Ok(ServerFeatures {
-            // electrum-client doesn't retain the hosts map data, but we already have it from the add_peer request
-            hosts: HashMap::new(),
-            genesis_hash: BlockHash::from_raw_hash(sha256d::Hash::from_byte_array(
-                features.genesis_hash,
-            )),
-            server_version: features.server_version,
-            protocol_min: features
-                .protocol_min
-                .parse()
-                .chain_err(|| "invalid protocol_min")?,
-            protocol_max: features
-                .protocol_max
-                .parse()
-                .chain_err(|| "invalid protocol_max")?,
-            pruning: features.pruning.map(|pruning| pruning as usize),
-            hash_function: features
-                .hash_function
-                .chain_err(|| "missing hash_function")?,
-        })
-    }
-}

src/electrum/discovery.rs

@@ -1,603 +0,0 @@
-use std::cmp::Ordering;
-use std::collections::{hash_map::Entry, BinaryHeap, HashMap, HashSet};
-use std::convert::TryInto;
-use std::fmt;
-use std::net::{IpAddr, SocketAddr, ToSocketAddrs};
-use std::str::FromStr;
-use std::sync::{Arc, RwLock};
-use std::thread;
-use std::time::{Duration, Instant};
-
-use electrum_client::ElectrumApi;
-
-use crate::chain::Network;
-use crate::electrum::{Client, Hostname, Port, ProtocolVersion, ServerFeatures};
-use crate::errors::{Result, ResultExt};
-use crate::util::spawn_thread;
-
-mod default_servers;
-use default_servers::add_default_servers;
-
-const HEALTH_CHECK_FREQ: Duration = Duration::from_secs(3600); // check servers every hour
-const JOB_INTERVAL: Duration = Duration::from_secs(1); // run one health check job every second
-const MAX_CONSECUTIVE_FAILURES: usize = 24; // drop servers after 24 consecutive failing attempts (~24 hours) (~24 hours)
-const MAX_QUEUE_SIZE: usize = 500; // refuse accepting new servers if we have that many health check jobs
-const MAX_SERVERS_PER_REQUEST: usize = 3; // maximum number of server hosts added per server.add_peer call
-const MAX_SERVICES_PER_REQUEST: usize = 6; // maximum number of services added per server.add_peer call
-
-#[derive(Debug)]
-pub struct DiscoveryManager {
-    /// A queue of scheduled health check jobs, including for healthy, unhealthy and untested servers
-    queue: RwLock<BinaryHeap<HealthCheck>>,
-
-    /// A list of servers that were found to be healthy on their last health check
-    healthy: RwLock<HashMap<ServerAddr, Server>>,
-
-    /// Used to test for protocol version compatibility
-    our_version: ProtocolVersion,
-
-    /// So that we don't list ourselves
-    our_addrs: HashSet<ServerAddr>,
-
-    /// For advertising ourself to other servers
-    our_features: ServerFeatures,
-
-    /// Whether we should announce ourselves to the servers we're connecting to
-    announce: bool,
-
-    /// Optional, will not support onion hosts without this
-    tor_proxy: Option<SocketAddr>,
-}
-
-/// A Server corresponds to a single IP address or onion hostname, with one or more services
-/// exposed on different ports.
-#[derive(Debug)]
-struct Server {
-    services: HashSet<Service>,
-    hostname: Hostname,
-    features: ServerFeatures,
-    // the `ServerAddr` isn't kept here directly, but is also available next to `Server` as the key for
-    // the `healthy` field on `DiscoveryManager`
-}
-
-#[derive(Eq, PartialEq, Hash, Clone, Debug)]
-enum ServerAddr {
-    Clearnet(IpAddr),
-    Onion(Hostname),
-}
-
-#[derive(Eq, PartialEq, Hash, Copy, Clone, Debug)]
-pub enum Service {
-    Tcp(Port),
-    Ssl(Port),
-    // unimplemented: Ws and Wss
-}
-
-/// A queued health check job, one per service/port (and not per server)
-#[derive(Eq, Debug)]
-struct HealthCheck {
-    addr: ServerAddr,
-    hostname: Hostname,
-    service: Service,
-    is_default: bool,
-    #[allow(dead_code)]
-    added_by: Option<IpAddr>,
-    last_check: Option<Instant>,
-    last_healthy: Option<Instant>,
-    consecutive_failures: usize,
-}
-
-/// The server entry format returned from server.peers.subscribe
-#[derive(Serialize)]
-pub struct ServerEntry(ServerAddr, Hostname, Vec<String>);
-
-impl DiscoveryManager {
-    pub fn new(
-        our_network: Network,
-        our_features: ServerFeatures,
-        our_version: ProtocolVersion,
-        announce: bool,
-        tor_proxy: Option<SocketAddr>,
-    ) -> Self {
-        let our_addrs = our_features
-            .hosts
-            .keys()
-            .filter_map(|hostname| {
-                ServerAddr::resolve(hostname)
-                    .map_err(|e| warn!("failed resolving own hostname {}: {:?}", hostname, e))
-                    .ok()
-            })
-            .collect();
-        let discovery = Self {
-            our_addrs,
-            our_version,
-            our_features,
-            announce,
-            tor_proxy,
-            healthy: Default::default(),
-            queue: Default::default(),
-        };
-        add_default_servers(&discovery, our_network);
-        discovery
-    }
-
-    /// Add a server requested via `server.add_peer`
-    pub fn add_server_request(&self, added_by: IpAddr, features: ServerFeatures) -> Result<()> {
-        self.verify_compatibility(&features)?;
-
-        let mut queue = self.queue.write().unwrap();
-        ensure!(queue.len() < MAX_QUEUE_SIZE, "queue size exceeded");
-
-        // TODO optimize
-        let mut existing_services: HashMap<ServerAddr, HashSet<Service>> = HashMap::new();
-        for job in queue.iter() {
-            existing_services
-                .entry(job.addr.clone())
-                .or_default()
-                .insert(job.service);
-        }
-
-        // collect HealthChecks for candidate services
-        let jobs = features
-            .hosts
-            .iter()
-            .take(MAX_SERVERS_PER_REQUEST)
-            .filter_map(|(hostname, ports)| {
-                let hostname = hostname.to_lowercase();
-
-                if hostname.len() > 100 {
-                    warn!("skipping invalid hostname");
-                    return None;
-                }
-                let addr = match ServerAddr::resolve(&hostname) {
-                    Ok(addr) => addr,
-                    Err(e) => {
-                        warn!("failed resolving {}: {:?}", hostname, e);
-                        return None;
-                    }
-                };
-                if !is_remote_addr(&addr) || self.our_addrs.contains(&addr) {
-                    warn!("skipping own or non-remote server addr");
-                    return None;
-                }
-                // ensure the server address matches the ip that advertised it to us.
-                // onion hosts are exempt.
-                if let ServerAddr::Clearnet(ip) = addr {
-                    if ip != added_by {
-                        warn!(
-                            "server ip does not match source ip ({}, {} != {})",
-                            hostname, ip, added_by
-                        );
-                        return None;
-                    }
-                }
-                Some((addr, hostname, ports))
-            })
-            .flat_map(|(addr, hostname, ports)| {
-                let tcp_service = ports.tcp_port.into_iter().map(Service::Tcp);
-                let ssl_service = ports.ssl_port.into_iter().map(Service::Ssl);
-                let services = tcp_service.chain(ssl_service).collect::<HashSet<Service>>();
-
-                services
-                    .into_iter()
-                    .filter(|service| {
-                        existing_services
-                            .get(&addr)
-                            .is_none_or(|s| !s.contains(service))
-                    })
-                    .map(|service| {
-                        HealthCheck::new(addr.clone(), hostname.clone(), service, Some(added_by))
-                    })
-                    .collect::<Vec<_>>()
-            })
-            .take(MAX_SERVICES_PER_REQUEST)
-            .collect::<Vec<_>>();
-
-        ensure!(
-            queue.len() + jobs.len() <= MAX_QUEUE_SIZE,
-            "queue size exceeded"
-        );
-
-        queue.extend(jobs);
-        Ok(())
-    }
-
-    /// Add a default server. Default servers are exempt from limits and given more leniency
-    /// before being removed due to unavailability.
-    pub fn add_default_server(&self, hostname: Hostname, services: Vec<Service>) -> Result<()> {
-        let addr = ServerAddr::resolve(&hostname)?;
-        let mut queue = self.queue.write().unwrap();
-        queue.extend(
-            services
-                .into_iter()
-                .map(|service| HealthCheck::new(addr.clone(), hostname.clone(), service, None)),
-        );
-        Ok(())
-    }
-
-    /// Get the list of healthy servers formatted for `servers.peers.subscribe`
-    pub fn get_servers(&self) -> Vec<ServerEntry> {
-        // XXX return a random sample instead of everything?
-        self.healthy
-            .read()
-            .unwrap()
-            .iter()
-            .map(|(addr, server)| {
-                ServerEntry(addr.clone(), server.hostname.clone(), server.feature_strs())
-            })
-            .collect()
-    }
-
-    pub fn our_features(&self) -> &ServerFeatures {
-        &self.our_features
-    }
-
-    /// Run the next health check in the queue (a single one)
-    fn run_health_check(&self) -> Result<()> {
-        // abort if there are no entries in the queue, or its still too early for the next one up
-        if self.queue.read().unwrap().peek().is_none_or(|next| {
-            next.last_check
-                .is_some_and(|t| t.elapsed() < HEALTH_CHECK_FREQ)
-        }) {
-            return Ok(());
-        }
-
-        let mut job = self.queue.write().unwrap().pop().unwrap();
-        debug!("processing {:?}", job);
-
-        let was_healthy = job.is_healthy();
-
-        match self.check_server(&job.addr, &job.hostname, job.service) {
-            Ok(features) => {
-                debug!("{} {:?} is available", job.hostname, job.service);
-
-                if !was_healthy {
-                    self.save_healthy_service(&job, features);
-                }
-                // XXX update features?
-
-                job.last_check = Some(Instant::now());
-                job.last_healthy = job.last_check;
-                job.consecutive_failures = 0;
-                // schedule the next health check
-                self.queue.write().unwrap().push(job);
-
-                Ok(())
-            }
-            Err(e) => {
-                debug!("{} {:?} is unavailable: {:?}", job.hostname, job.service, e);
-
-                if was_healthy {
-                    // XXX should we assume the server's other services are down too?
-                    self.remove_unhealthy_service(&job);
-                }
-
-                job.last_check = Some(Instant::now());
-                job.consecutive_failures += 1;
-
-                if job.should_retry() {
-                    self.queue.write().unwrap().push(job);
-                } else {
-                    debug!("giving up on {:?}", job);
-                }
-
-                Err(e)
-            }
-        }
-    }
-
-    /// Upsert the server/service into the healthy set
-    fn save_healthy_service(&self, job: &HealthCheck, features: ServerFeatures) {
-        let addr = job.addr.clone();
-        let mut healthy = self.healthy.write().unwrap();
-        healthy
-            .entry(addr)
-            .or_insert_with(|| Server::new(job.hostname.clone(), features))
-            .services
-            .insert(job.service);
-    }
-
-    /// Remove the service, and remove the server entirely if it has no other reamining healthy services
-    fn remove_unhealthy_service(&self, job: &HealthCheck) {
-        let addr = job.addr.clone();
-        let mut healthy = self.healthy.write().unwrap();
-        if let Entry::Occupied(mut entry) = healthy.entry(addr) {
-            let server = entry.get_mut();
-            assert!(server.services.remove(&job.service));
-            if server.services.is_empty() {
-                entry.remove_entry();
-            }
-        } else {
-            unreachable!("missing expected server, corrupted state");
-        }
-    }
-
-    fn check_server(
-        &self,
-        addr: &ServerAddr,
-        hostname: &Hostname,
-        service: Service,
-    ) -> Result<ServerFeatures> {
-        debug!("checking service {:?} {:?}", addr, service);
-
-        let server_url = match (addr, service) {
-            (ServerAddr::Clearnet(ip), Service::Tcp(port)) => format!("tcp://{}:{}", ip, port),
-            (ServerAddr::Clearnet(_), Service::Ssl(port)) => format!("ssl://{}:{}", hostname, port),
-            (ServerAddr::Onion(onion_host), Service::Tcp(port)) => {
-                format!("tcp://{}:{}", onion_host, port)
-            }
-            (ServerAddr::Onion(onion_host), Service::Ssl(port)) => {
-                format!("ssl://{}:{}", onion_host, port)
-            }
-        };
-
-        let mut config = electrum_client::ConfigBuilder::new();
-        if let ServerAddr::Onion(_) = addr {
-            let socks = electrum_client::Socks5Config::new(
-                self.tor_proxy
-                    .chain_err(|| "no tor proxy configured, onion hosts are unsupported")?,
-            );
-            config = config.socks5(Some(socks))
-        }
-
-        let client = Client::from_config(&server_url, config.build())?;
-
-        let features = client.server_features()?.try_into()?;
-        self.verify_compatibility(&features)?;
-
-        if self.announce {
-            // XXX should we require the other side to reciprocate?
-            ensure!(
-                client.server_add_peer(&self.our_features)?,
-                "server does not reciprocate"
-            );
-        }
-
-        Ok(features)
-    }
-
-    fn verify_compatibility(&self, features: &ServerFeatures) -> Result<()> {
-        ensure!(
-            features.genesis_hash == self.our_features.genesis_hash,
-            "incompatible networks"
-        );
-
-        ensure!(
-            features.protocol_min <= self.our_version && features.protocol_max >= self.our_version,
-            "incompatible protocol versions"
-        );
-
-        ensure!(
-            features.hash_function == "sha256",
-            "incompatible hash function"
-        );
-
-        Ok(())
-    }
-
-    pub fn spawn_jobs_thread(manager: Arc<DiscoveryManager>) {
-        spawn_thread("discovery-jobs", move || loop {
-            if let Err(e) = manager.run_health_check() {
-                debug!("health check failed: {:?}", e);
-            }
-            // XXX use a dynamic JOB_INTERVAL, adjusted according to the queue size and HEALTH_CHECK_FREQ?
-            thread::sleep(JOB_INTERVAL);
-        });
-    }
-}
-
-impl Server {
-    fn new(hostname: Hostname, features: ServerFeatures) -> Self {
-        Server {
-            hostname,
-            features,
-            services: HashSet::new(),
-        }
-    }
-
-    /// Get server features and services in the compact string array format used for `servers.peers.subscribe`
-    fn feature_strs(&self) -> Vec<String> {
-        let mut strs = Vec::with_capacity(self.services.len() + 1);
-        strs.push(format!("v{}", self.features.protocol_max));
-        if let Some(pruning) = self.features.pruning {
-            strs.push(format!("p{}", pruning));
-        }
-        strs.extend(self.services.iter().map(|s| s.to_string()));
-        strs
-    }
-}
-
-impl ServerAddr {
-    fn resolve(host: &str) -> Result<Self> {
-        Ok(if host.ends_with(".onion") {
-            ServerAddr::Onion(host.into())
-        } else if let Ok(ip) = IpAddr::from_str(host) {
-            ServerAddr::Clearnet(ip)
-        } else {
-            let ip = format!("{}:1", host)
-                .to_socket_addrs()
-                .chain_err(|| "hostname resolution failed")?
-                .next()
-                .chain_err(|| "hostname resolution failed")?
-                .ip();
-            ServerAddr::Clearnet(ip)
-        })
-    }
-}
-
-impl fmt::Display for ServerAddr {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            ServerAddr::Clearnet(ip) => write!(f, "{}", ip),
-            ServerAddr::Onion(hostname) => write!(f, "{}", hostname),
-        }
-    }
-}
-
-impl serde::Serialize for ServerAddr {
-    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        serializer.serialize_str(&self.to_string())
-    }
-}
-
-impl HealthCheck {
-    fn new(
-        addr: ServerAddr,
-        hostname: Hostname,
-        service: Service,
-        added_by: Option<IpAddr>,
-    ) -> Self {
-        HealthCheck {
-            addr,
-            hostname,
-            service,
-            is_default: added_by.is_none(),
-            added_by,
-            last_check: None,
-            last_healthy: None,
-            consecutive_failures: 0,
-        }
-    }
-
-    fn is_healthy(&self) -> bool {
-        match (self.last_check, self.last_healthy) {
-            (Some(last_check), Some(last_healthy)) => last_check == last_healthy,
-            _ => false,
-        }
-    }
-
-    // allow the server to fail up to MAX_CONSECTIVE_FAILURES time before giving up on it.
-    // if its a non-default server and the very first attempt fails, give up immediatly.
-    fn should_retry(&self) -> bool {
-        (self.last_healthy.is_some() || self.is_default)
-            && self.consecutive_failures < MAX_CONSECUTIVE_FAILURES
-    }
-}
-
-impl PartialEq for HealthCheck {
-    fn eq(&self, other: &Self) -> bool {
-        self.hostname == other.hostname && self.service == other.service
-    }
-}
-
-impl Ord for HealthCheck {
-    fn cmp(&self, other: &Self) -> Ordering {
-        self.last_check.cmp(&other.last_check).reverse()
-    }
-}
-
-impl PartialOrd for HealthCheck {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl fmt::Display for Service {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Service::Tcp(port) => write!(f, "t{}", port),
-            Service::Ssl(port) => write!(f, "s{}", port),
-        }
-    }
-}
-
-fn is_remote_addr(addr: &ServerAddr) -> bool {
-    match addr {
-        ServerAddr::Onion(_) => true,
-        ServerAddr::Clearnet(ip) => {
-            !ip.is_loopback()
-                && !ip.is_unspecified()
-                && !ip.is_multicast()
-                && !match ip {
-                    IpAddr::V4(ipv4) => ipv4.is_private(),
-                    IpAddr::V6(_) => false,
-                }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::chain::genesis_hash;
-    use crate::chain::Network;
-    use std::time;
-
-    use crate::config::VERSION_STRING;
-
-    const PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::new(1, 4);
-
-    #[test]
-    #[ignore = "This test requires external connection to server that no longer exists"]
-    fn test() -> Result<()> {
-        stderrlog::new().verbosity(4).init().unwrap();
-
-        #[cfg(feature = "liquid")]
-        let testnet = Network::LiquidTestnet;
-        #[cfg(not(feature = "liquid"))]
-        let testnet = Network::Testnet;
-
-        let features = ServerFeatures {
-            hosts: serde_json::from_str("{\"test.foobar.example\":{\"tcp_port\":60002}}").unwrap(),
-            server_version: VERSION_STRING.clone(),
-            genesis_hash: genesis_hash(testnet),
-            protocol_min: PROTOCOL_VERSION,
-            protocol_max: PROTOCOL_VERSION,
-            hash_function: "sha256".into(),
-            pruning: None,
-        };
-        let discovery = Arc::new(DiscoveryManager::new(
-            testnet,
-            features,
-            PROTOCOL_VERSION,
-            false,
-            None,
-        ));
-        discovery
-            .add_default_server(
-                "electrum.blockstream.info".into(),
-                vec![Service::Tcp(60001)],
-            )
-            .unwrap();
-        discovery
-            .add_default_server("testnet.hsmiths.com".into(), vec![Service::Ssl(53012)])
-            .unwrap();
-        discovery
-            .add_default_server(
-                "tn.not.fyi".into(),
-                vec![Service::Tcp(55001), Service::Ssl(55002)],
-            )
-            .unwrap();
-        discovery
-            .add_default_server(
-                "electrum.blockstream.info".into(),
-                vec![Service::Tcp(60001), Service::Ssl(60002)],
-            )
-            .unwrap();
-        discovery
-            .add_default_server(
-                "explorerzydxu5ecjrkwceayqybizmpjjznk5izmitf2modhcusuqlid.onion".into(),
-                vec![Service::Tcp(143)],
-            )
-            .unwrap();
-
-        debug!("{:#?}", discovery);
-
-        for _ in 0..12 {
-            discovery
-                .run_health_check()
-                .map_err(|e| warn!("{:?}", e))
-                .ok();
-            thread::sleep(time::Duration::from_secs(1));
-        }
-
-        debug!("{:#?}", discovery);
-
-        info!("{}", json!(discovery.get_servers()));
-
-        Ok(())
-    }
-}

src/electrum/discovery/default_servers.rs

@@ -1,449 +0,0 @@
-use crate::chain::Network;
-#[allow(unused_imports)]
-use crate::electrum::discovery::{DiscoveryManager, Service};
-
-#[allow(unused_variables)]
-pub fn add_default_servers(discovery: &DiscoveryManager, network: Network) {
-    match network {
-        #[cfg(not(feature = "liquid"))]
-        Network::Bitcoin => {
-            discovery
-                .add_default_server(
-                    "3smoooajg7qqac2y.onion".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "81-7-10-251.blue.kundencontroller.de".into(),
-                    vec![Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "E-X.not.fyi".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "VPS.hsmiths.com".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "b.ooze.cc".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "bauerjda5hnedjam.onion".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "bauerjhejlv6di7s.onion".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "bitcoin.corgi.party".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "bitcoin3nqy3db7c.onion".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "bitcoins.sk".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "btc.cihar.com".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "btc.xskyx.net".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "currentlane.lovebitco.in".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "daedalus.bauerj.eu".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum.jochen-hoenicke.de".into(),
-                    vec![Service::Tcp(50003), Service::Ssl(50005)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "dragon085.startdedicated.de".into(),
-                    vec![Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "e-1.claudioboxx.com".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "e.keff.org".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum-server.ninja".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum-unlimited.criptolayer.net".into(),
-                    vec![Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum.eff.ro".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum.festivaldelhumor.org".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum.hsmiths.com".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum.leblancnet.us".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server("electrum.mindspot.org".into(), vec![Service::Ssl(50002)])
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum.qtornado.com".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server("electrum.taborsky.cz".into(), vec![Service::Ssl(50002)])
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum.villocq.com".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum2.eff.ro".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum2.villocq.com".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrumx.bot.nu".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrumx.ddns.net".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server("electrumx.ftp.sh".into(), vec![Service::Ssl(50002)])
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrumx.ml".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrumx.soon.it".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server("electrumxhqdsmlu.onion".into(), vec![Service::Tcp(50001)])
-                .ok();
-            discovery
-                .add_default_server(
-                    "elx01.knas.systems".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "enode.duckdns.org".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "fedaykin.goip.de".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "fn.48.org".into(),
-                    vec![Service::Tcp(50003), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "helicarrier.bauerj.eu".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "hsmiths4fyqlw5xw.onion".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "hsmiths5mjk6uijs.onion".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "icarus.tetradrachm.net".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrum.emzy.de".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "ndnd.selfhost.eu".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server("ndndword5lpb7eex.onion".into(), vec![Service::Tcp(50001)])
-                .ok();
-            discovery
-                .add_default_server(
-                    "orannis.com".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "ozahtqwp25chjdjd.onion".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "qtornadoklbgdyww.onion".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server("rbx.curalle.ovh".into(), vec![Service::Ssl(50002)])
-                .ok();
-            discovery
-                .add_default_server("s7clinmo4cazmhul.onion".into(), vec![Service::Tcp(50001)])
-                .ok();
-            discovery
-                .add_default_server(
-                    "tardis.bauerj.eu".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server("technetium.network".into(), vec![Service::Ssl(50002)])
-                .ok();
-            discovery
-                .add_default_server(
-                    "tomscryptos.com".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "ulrichard.ch".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "vmd27610.contaboserver.net".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "vmd30612.contaboserver.net".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "wsw6tua3xl24gsmi264zaep6seppjyrkyucpsmuxnjzyt3f3j6swshad.onion".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "xray587.startdedicated.de".into(),
-                    vec![Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "yuio.top".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "bitcoin.dragon.zone".into(),
-                    vec![Service::Tcp(50003), Service::Ssl(50004)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "ecdsa.net".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(110)],
-                )
-                .ok();
-            discovery
-                .add_default_server("btc.usebsv.com".into(), vec![Service::Ssl(50006)])
-                .ok();
-            discovery
-                .add_default_server(
-                    "e2.keff.org".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server("electrum.hodlister.co".into(), vec![Service::Ssl(50002)])
-                .ok();
-            discovery
-                .add_default_server("electrum3.hodlister.co".into(), vec![Service::Ssl(50002)])
-                .ok();
-            discovery
-                .add_default_server("electrum5.hodlister.co".into(), vec![Service::Ssl(50002)])
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrumx.electricnewyear.net".into(),
-                    vec![Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "fortress.qtornado.com".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(443)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "green-gold.westeurope.cloudapp.azure.com".into(),
-                    vec![Service::Tcp(56001), Service::Ssl(56002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "electrumx.erbium.eu".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-        }
-        #[cfg(not(feature = "liquid"))]
-        Network::Testnet => {
-            discovery
-                .add_default_server(
-                    "hsmithsxurybd7uh.onion".into(),
-                    vec![Service::Tcp(53011), Service::Ssl(53012)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "testnet.hsmiths.com".into(),
-                    vec![Service::Tcp(53011), Service::Ssl(53012)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "testnet.qtornado.com".into(),
-                    vec![Service::Tcp(51001), Service::Ssl(51002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "testnet1.bauerj.eu".into(),
-                    vec![Service::Tcp(50001), Service::Ssl(50002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "tn.not.fyi".into(),
-                    vec![Service::Tcp(55001), Service::Ssl(55002)],
-                )
-                .ok();
-            discovery
-                .add_default_server(
-                    "bitcoin.cluelessperson.com".into(),
-                    vec![Service::Tcp(51001), Service::Ssl(51002)],
-                )
-                .ok();
-        }
-
-        _ => (),
-    }
-}

src/electrum/mod.rs

@@ -1,119 +0,0 @@
-mod server;
-pub use server::RPC;
-
-#[cfg(feature = "electrum-discovery")]
-mod client;
-#[cfg(feature = "electrum-discovery")]
-mod discovery;
-#[cfg(feature = "electrum-discovery")]
-pub use {client::Client, discovery::DiscoveryManager};
-
-use std::cmp::Ordering;
-use std::collections::HashMap;
-use std::str::FromStr;
-
-use serde::{de, Deserialize, Deserializer, Serialize};
-
-use crate::chain::BlockHash;
-use crate::errors::ResultExt;
-use crate::util::BlockId;
-
-pub fn get_electrum_height(blockid: Option<BlockId>, has_unconfirmed_parents: bool) -> isize {
-    match (blockid, has_unconfirmed_parents) {
-        (Some(blockid), _) => blockid.height as isize,
-        (None, false) => 0,
-        (None, true) => -1,
-    }
-}
-
-pub type Port = u16;
-pub type Hostname = String;
-
-pub type ServerHosts = HashMap<Hostname, ServerPorts>;
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct ServerFeatures {
-    pub hosts: ServerHosts,
-    pub genesis_hash: BlockHash,
-    pub server_version: String,
-    pub protocol_min: ProtocolVersion,
-    pub protocol_max: ProtocolVersion,
-    pub pruning: Option<usize>,
-    pub hash_function: String,
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct ServerPorts {
-    tcp_port: Option<Port>,
-    ssl_port: Option<Port>,
-}
-
-#[derive(Eq, PartialEq, Debug, Clone, Default)]
-pub struct ProtocolVersion {
-    major: usize,
-    minor: usize,
-}
-
-impl ProtocolVersion {
-    pub const fn new(major: usize, minor: usize) -> Self {
-        Self { major, minor }
-    }
-}
-
-impl Ord for ProtocolVersion {
-    fn cmp(&self, other: &Self) -> Ordering {
-        self.major
-            .cmp(&other.major)
-            .then_with(|| self.minor.cmp(&other.minor))
-    }
-}
-
-impl PartialOrd for ProtocolVersion {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl FromStr for ProtocolVersion {
-    type Err = crate::errors::Error;
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let mut iter = s.split('.');
-        Ok(Self {
-            major: iter
-                .next()
-                .chain_err(|| "missing major")?
-                .parse()
-                .chain_err(|| "invalid major")?,
-            minor: iter
-                .next()
-                .chain_err(|| "missing minor")?
-                .parse()
-                .chain_err(|| "invalid minor")?,
-        })
-    }
-}
-
-impl std::fmt::Display for ProtocolVersion {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}.{}", self.major, self.minor)
-    }
-}
-
-impl Serialize for ProtocolVersion {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        serializer.collect_str(&self)
-    }
-}
-
-impl<'de> Deserialize<'de> for ProtocolVersion {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let s = String::deserialize(deserializer)?;
-        FromStr::from_str(&s).map_err(de::Error::custom)
-    }
-}

src/elements/asset.rs

@@ -1,662 +0,0 @@
-use std::collections::{HashMap, HashSet};
-use std::sync::{Arc, RwLock, RwLockReadGuard};
-
-use bitcoin::hashes::{sha256, Hash};
-use elements::confidential::{Asset, Value};
-use elements::encode::{deserialize, serialize};
-use elements::secp256k1_zkp::ZERO_TWEAK;
-use elements::{issuance::ContractHash, AssetId, AssetIssuance, OutPoint, Transaction, TxIn};
-
-use crate::chain::{BNetwork, BlockHash, Network, Txid};
-use crate::elements::peg::{get_pegin_data, get_pegout_data, PeginInfo, PegoutInfo};
-use crate::elements::registry::{AssetMeta, AssetRegistry};
-use crate::errors::*;
-use crate::new_index::schema::{Operation, TxHistoryInfo, TxHistoryKey, TxHistoryRow};
-use crate::new_index::{db::DBFlush, ChainQuery, DBRow, Mempool, Query};
-use crate::util::{
-    bincode_util, full_hash, Bytes, FullHash, IsProvablyUnspendable, TransactionStatus, TxInput,
-};
-
-lazy_static! {
-    pub static ref NATIVE_ASSET_ID: AssetId =
-        "6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d"
-            .parse()
-            .unwrap();
-    pub static ref NATIVE_ASSET_ID_TESTNET: AssetId =
-        "144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49"
-            .parse()
-            .unwrap();
-    pub static ref NATIVE_ASSET_ID_REGTEST: AssetId =
-        "5ac9f65c0efcc4775e0baec4ec03abdde22473cd3cf33c0419ca290e0751b225"
-            .parse()
-            .unwrap();
-}
-
-fn parse_asset_id(sl: &[u8]) -> AssetId {
-    AssetId::from_slice(sl).expect("failed to parse AssetId")
-}
-
-#[derive(Serialize)]
-#[serde(untagged)]
-#[allow(clippy::large_enum_variant)]
-pub enum LiquidAsset {
-    Issued(IssuedAsset),
-    Native(PeggedAsset),
-}
-
-#[derive(Serialize)]
-pub struct PeggedAsset {
-    pub asset_id: AssetId,
-    pub chain_stats: PeggedAssetStats,
-    pub mempool_stats: PeggedAssetStats,
-}
-
-#[derive(Serialize)]
-pub struct IssuedAsset {
-    pub asset_id: AssetId,
-    pub issuance_txin: TxInput,
-    #[serde(serialize_with = "crate::util::serialize_outpoint")]
-    pub issuance_prevout: OutPoint,
-    pub reissuance_token: AssetId,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub contract_hash: Option<ContractHash>,
-
-    // the confirmation status of the initial issuance transaction
-    pub status: TransactionStatus,
-
-    pub chain_stats: IssuedAssetStats,
-    pub mempool_stats: IssuedAssetStats,
-
-    // optional metadata from registry
-    #[serde(flatten)]
-    pub meta: Option<AssetMeta>,
-}
-
-// DB representation (issued assets only)
-#[derive(Serialize, Deserialize, Debug)]
-pub struct AssetRow {
-    pub issuance_txid: FullHash,
-    pub issuance_vin: u32,
-    pub prev_txid: FullHash,
-    pub prev_vout: u32,
-    pub issuance: Bytes, // bincode does not like dealing with AssetIssuance, deserialization fails with "invalid type: sequence, expected a struct"
-    pub reissuance_token: FullHash,
-}
-
-impl IssuedAsset {
-    pub fn new(
-        asset_id: &AssetId,
-        asset: &AssetRow,
-        (chain_stats, mempool_stats): (IssuedAssetStats, IssuedAssetStats),
-        meta: Option<AssetMeta>,
-        status: TransactionStatus,
-    ) -> Self {
-        let issuance: AssetIssuance =
-            deserialize(&asset.issuance).expect("failed parsing AssetIssuance");
-
-        let reissuance_token = parse_asset_id(&asset.reissuance_token);
-
-        let contract_hash = if issuance.asset_entropy != [0u8; 32] {
-            Some(ContractHash::from_byte_array(issuance.asset_entropy))
-        } else {
-            None
-        };
-
-        Self {
-            asset_id: *asset_id,
-            issuance_txin: TxInput {
-                txid: deserialize(&asset.issuance_txid).unwrap(),
-                vin: asset.issuance_vin,
-            },
-            issuance_prevout: OutPoint {
-                txid: deserialize(&asset.prev_txid).unwrap(),
-                vout: asset.prev_vout,
-            },
-            contract_hash,
-            reissuance_token,
-            status,
-            chain_stats,
-            mempool_stats,
-            meta,
-        }
-    }
-}
-
-impl LiquidAsset {
-    pub fn supply(&self) -> Option<u64> {
-        match self {
-            LiquidAsset::Native(asset) => Some(
-                asset.chain_stats.peg_in_amount
-                    - asset.chain_stats.peg_out_amount
-                    - asset.chain_stats.burned_amount
-                    + asset.mempool_stats.peg_in_amount
-                    - asset.mempool_stats.peg_out_amount
-                    - asset.mempool_stats.burned_amount,
-            ),
-            LiquidAsset::Issued(asset) => {
-                if asset.chain_stats.has_blinded_issuances
-                    || asset.mempool_stats.has_blinded_issuances
-                {
-                    None
-                } else {
-                    Some(
-                        asset.chain_stats.issued_amount - asset.chain_stats.burned_amount
-                            + asset.mempool_stats.issued_amount
-                            - asset.mempool_stats.burned_amount,
-                    )
-                }
-            }
-        }
-    }
-    pub fn precision(&self) -> u8 {
-        match self {
-            LiquidAsset::Native(_) => 8,
-            LiquidAsset::Issued(asset) => asset.meta.as_ref().map_or(0, |m| m.precision),
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-#[cfg_attr(test, derive(PartialEq, Eq))]
-pub struct IssuingInfo {
-    pub txid: FullHash,
-    pub vin: u32,
-    pub is_reissuance: bool,
-    // None for blinded issuances
-    pub issued_amount: Option<u64>,
-    pub token_amount: Option<u64>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-#[cfg_attr(test, derive(PartialEq, Eq))]
-pub struct BurningInfo {
-    pub txid: FullHash,
-    pub vout: u32,
-    pub value: u64,
-}
-
-// Index confirmed transaction issuances and save as db rows
-pub fn index_confirmed_tx_assets(
-    tx: &Transaction,
-    confirmed_height: u32,
-    tx_position: u16,
-    network: Network,
-    parent_network: BNetwork,
-    rows: &mut Vec<DBRow>,
-    op: &Operation,
-) {
-    let (history, issuances) = index_tx_assets(tx, network, parent_network);
-
-    rows.extend(history.into_iter().map(|(asset_id, info)| {
-        let history_row = asset_history_row(&asset_id, confirmed_height, tx_position, info);
-        if let Operation::DeleteBlocksWithHistory(tx) = op {
-            tx.send(history_row.key.hash)
-                .expect("unbounded channel won't fail");
-        }
-        history_row.into_row()
-    }));
-
-    // the initial issuance is kept twice: once in the history index under I<asset><height><txid:vin>,
-    // and once separately under i<asset> for asset lookup with some more associated metadata.
-    // reissuances are only kept under the history index.
-    rows.extend(issuances.into_iter().map(|(asset_id, asset_row)| DBRow {
-        key: [b"i", &asset_id.into_inner()[..]].concat(),
-        value: bincode_util::serialize_little(&asset_row).unwrap(),
-    }));
-}
-
-// Index mempool transaction issuances and save to in-memory store
-pub fn index_mempool_tx_assets(
-    tx: &Transaction,
-    network: Network,
-    parent_network: BNetwork,
-    asset_history: &mut HashMap<AssetId, Vec<TxHistoryInfo>>,
-    asset_issuance: &mut HashMap<AssetId, AssetRow>,
-) {
-    let (history, issuances) = index_tx_assets(tx, network, parent_network);
-    for (asset_id, info) in history {
-        asset_history.entry(asset_id).or_default().push(info);
-    }
-    for (asset_id, issuance) in issuances {
-        asset_issuance.insert(asset_id, issuance);
-    }
-}
-
-// Remove mempool transaction issuances from in-memory store
-pub fn remove_mempool_tx_assets(
-    to_remove: &HashSet<&Txid>,
-    asset_history: &mut HashMap<AssetId, Vec<TxHistoryInfo>>,
-    asset_issuance: &mut HashMap<AssetId, AssetRow>,
-) {
-    // TODO optimize
-    asset_history.retain(|_assethash, entries| {
-        entries.retain(|entry| !to_remove.contains(&entry.get_txid()));
-        !entries.is_empty()
-    });
-
-    asset_issuance.retain(|_assethash, issuance| {
-        let txid: Txid = deserialize(&issuance.issuance_txid).unwrap();
-        !to_remove.contains(&txid)
-    });
-}
-
-type HistoryAndIssuances = (Vec<(AssetId, TxHistoryInfo)>, Vec<(AssetId, AssetRow)>);
-// Internal utility function, index a transaction and return its history entries and issuances
-fn index_tx_assets(
-    tx: &Transaction,
-    network: Network,
-    parent_network: BNetwork,
-) -> HistoryAndIssuances {
-    let mut history = vec![];
-    let mut issuances = vec![];
-
-    let txid = full_hash(&tx.txid()[..]);
-
-    for (txo_index, txo) in tx.output.iter().enumerate() {
-        if let Some(pegout) = get_pegout_data(txo, network, parent_network) {
-            history.push((
-                pegout.asset.explicit().unwrap(),
-                TxHistoryInfo::Pegout(PegoutInfo {
-                    txid,
-                    vout: txo_index as u32,
-                    value: pegout.value,
-                }),
-            ));
-        } else if txo.script_pubkey.is_provably_unspendable_() && !txo.is_fee() {
-            if let (Asset::Explicit(asset_id), Value::Explicit(value)) = (txo.asset, txo.value) {
-                if value > 0 {
-                    history.push((
-                        asset_id,
-                        TxHistoryInfo::Burning(BurningInfo {
-                            txid,
-                            vout: txo_index as u32,
-                            value,
-                        }),
-                    ));
-                }
-            }
-        }
-    }
-
-    for (txi_index, txi) in tx.input.iter().enumerate() {
-        if let Some(pegin) = get_pegin_data(txi, network) {
-            history.push((
-                pegin.asset,
-                TxHistoryInfo::Pegin(PeginInfo {
-                    txid,
-                    vin: txi_index as u32,
-                    value: pegin.value,
-                }),
-            ));
-        } else if txi.has_issuance() {
-            let is_reissuance = txi.asset_issuance.asset_blinding_nonce != ZERO_TWEAK;
-
-            let asset_entropy = get_issuance_entropy(txi).expect("invalid issuance");
-            let asset_id = AssetId::from_entropy(asset_entropy);
-
-            let issued_amount = match txi.asset_issuance.amount {
-                Value::Explicit(amount) => Some(amount),
-                Value::Null => Some(0),
-                _ => None,
-            };
-            let token_amount = match txi.asset_issuance.inflation_keys {
-                Value::Explicit(amount) => Some(amount),
-                Value::Null => Some(0),
-                _ => None,
-            };
-
-            history.push((
-                asset_id,
-                TxHistoryInfo::Issuing(IssuingInfo {
-                    txid,
-                    vin: txi_index as u32,
-                    is_reissuance,
-                    issued_amount,
-                    token_amount,
-                }),
-            ));
-
-            if !is_reissuance {
-                let is_confidential =
-                    matches!(txi.asset_issuance.inflation_keys, Value::Confidential(..));
-                let reissuance_token =
-                    AssetId::reissuance_token_from_entropy(asset_entropy, is_confidential);
-
-                issuances.push((
-                    asset_id,
-                    AssetRow {
-                        issuance_txid: txid,
-                        issuance_vin: txi_index as u32,
-                        prev_txid: full_hash(&txi.previous_output.txid[..]),
-                        prev_vout: txi.previous_output.vout,
-                        issuance: serialize(&txi.asset_issuance),
-                        reissuance_token: full_hash(&reissuance_token.into_inner()[..]),
-                    },
-                ));
-            }
-        }
-    }
-
-    (history, issuances)
-}
-
-fn asset_history_row(
-    asset_id: &AssetId,
-    confirmed_height: u32,
-    tx_position: u16,
-    txinfo: TxHistoryInfo,
-) -> TxHistoryRow {
-    let key = TxHistoryKey {
-        code: b'I',
-        hash: full_hash(&asset_id.into_inner()[..]),
-        confirmed_height,
-        tx_position,
-        txinfo,
-    };
-    TxHistoryRow { key }
-}
-
-pub enum AssetRegistryLock<'a> {
-    RwLock(&'a Arc<RwLock<AssetRegistry>>),
-    RwLockReadGuard(&'a RwLockReadGuard<'a, AssetRegistry>),
-}
-
-pub fn lookup_asset(
-    query: &Query,
-    registry: Option<AssetRegistryLock>,
-    asset_id: &AssetId,
-    meta: Option<&AssetMeta>, // may optionally be provided if already known
-) -> Result<Option<LiquidAsset>> {
-    if query.network().pegged_asset() == Some(asset_id) {
-        let (chain_stats, mempool_stats) = pegged_asset_stats(query, asset_id);
-
-        return Ok(Some(LiquidAsset::Native(PeggedAsset {
-            asset_id: *asset_id,
-            chain_stats,
-            mempool_stats,
-        })));
-    }
-
-    let history_db = query.chain().store().history_db();
-    let mempool = query.mempool();
-    let mempool_issuances = &mempool.asset_issuance;
-
-    let chain_row = history_db
-        .get(&[b"i", &asset_id.into_inner()[..]].concat())
-        .map(|row| {
-            bincode_util::deserialize_little::<AssetRow>(&row).expect("failed parsing AssetRow")
-        });
-
-    let row = chain_row
-        .as_ref()
-        .or_else(|| mempool_issuances.get(asset_id));
-
-    Ok(if let Some(row) = row {
-        let reissuance_token = parse_asset_id(&row.reissuance_token);
-
-        let meta = meta.cloned().or_else(|| match registry {
-            Some(AssetRegistryLock::RwLock(rwlock)) => {
-                rwlock.read().unwrap().get(asset_id).cloned()
-            }
-            Some(AssetRegistryLock::RwLockReadGuard(guard)) => guard.get(asset_id).cloned(),
-            None => None,
-        });
-        let stats = issued_asset_stats(query.chain(), &mempool, asset_id, &reissuance_token);
-        let status = query.get_tx_status(&deserialize(&row.issuance_txid).unwrap());
-
-        let asset = IssuedAsset::new(asset_id, row, stats, meta, status);
-
-        Some(LiquidAsset::Issued(asset))
-    } else {
-        None
-    })
-}
-
-pub fn get_issuance_entropy(txin: &TxIn) -> Result<sha256::Midstate> {
-    if !txin.has_issuance() {
-        bail!("input has no issuance");
-    }
-
-    let is_reissuance = txin.asset_issuance.asset_blinding_nonce != ZERO_TWEAK;
-
-    Ok(if !is_reissuance {
-        let contract_hash = ContractHash::from_slice(&txin.asset_issuance.asset_entropy)
-            .chain_err(|| "invalid entropy (contract hash)")?;
-        AssetId::generate_asset_entropy(txin.previous_output, contract_hash)
-    } else {
-        sha256::Midstate::from_slice(&txin.asset_issuance.asset_entropy)
-            .chain_err(|| "invalid entropy (reissuance)")?
-    })
-}
-
-//
-// Asset stats
-//
-
-#[derive(Serialize, Deserialize, Debug, Default)]
-pub struct IssuedAssetStats {
-    pub tx_count: usize,
-    pub issuance_count: usize,
-    pub issued_amount: u64,
-    pub burned_amount: u64,
-    pub has_blinded_issuances: bool,
-    pub reissuance_tokens: Option<u64>, // none if confidential
-    pub burned_reissuance_tokens: u64,
-}
-
-#[derive(Serialize, Deserialize, Debug, Default)]
-pub struct PeggedAssetStats {
-    pub tx_count: usize,
-    pub peg_in_count: usize,
-    pub peg_in_amount: u64,
-    pub peg_out_count: usize,
-    pub peg_out_amount: u64,
-    pub burn_count: usize,
-    pub burned_amount: u64,
-}
-
-type AssetStatApplyFn<T> = fn(&TxHistoryInfo, &mut T, &mut HashSet<Txid>);
-
-fn asset_cache_key(asset_id: &AssetId) -> Bytes {
-    [b"z", &asset_id.into_inner()[..]].concat()
-}
-
-fn asset_cache_row<T>(asset_id: &AssetId, stats: &T, blockhash: &BlockHash) -> DBRow
-where
-    T: serde::Serialize,
-{
-    DBRow {
-        key: asset_cache_key(asset_id),
-        value: bincode_util::serialize_little(&(stats, blockhash)).unwrap(),
-    }
-}
-
-// Get stats for the network's pegged asset
-fn pegged_asset_stats(query: &Query, asset_id: &AssetId) -> (PeggedAssetStats, PeggedAssetStats) {
-    (
-        chain_asset_stats(query.chain(), asset_id, apply_pegged_asset_stats),
-        mempool_asset_stats(&query.mempool(), asset_id, apply_pegged_asset_stats),
-    )
-}
-
-// Get stats for issued assets
-fn issued_asset_stats(
-    chain: &ChainQuery,
-    mempool: &Mempool,
-    asset_id: &AssetId,
-    reissuance_token: &AssetId,
-) -> (IssuedAssetStats, IssuedAssetStats) {
-    let afn = apply_issued_asset_stats;
-
-    let mut chain_stats = chain_asset_stats(chain, asset_id, afn);
-    chain_stats.burned_reissuance_tokens =
-        chain_asset_stats(chain, reissuance_token, afn).burned_amount;
-
-    let mut mempool_stats = mempool_asset_stats(mempool, asset_id, afn);
-    mempool_stats.burned_reissuance_tokens =
-        mempool_asset_stats(mempool, reissuance_token, afn).burned_amount;
-
-    (chain_stats, mempool_stats)
-}
-
-// Get on-chain confirmed asset stats (issued or the pegged asset)
-fn chain_asset_stats<T>(chain: &ChainQuery, asset_id: &AssetId, apply_fn: AssetStatApplyFn<T>) -> T
-where
-    T: Default + serde::Serialize + serde::de::DeserializeOwned,
-{
-    // get the last known stats and the blockhash they are updated for.
-    // invalidates the cache if the block was orphaned.
-    let cache: Option<(T, usize)> = chain
-        .store()
-        .cache_db()
-        .get(&asset_cache_key(asset_id))
-        .map(|c| bincode_util::deserialize_little(&c).unwrap())
-        .and_then(|(stats, blockhash)| {
-            chain
-                .height_by_hash(&blockhash)
-                .map(|height| (stats, height))
-        });
-
-    // update stats with new transactions since
-    let (newstats, lastblock) = cache.map_or_else(
-        || chain_asset_stats_delta(chain, asset_id, T::default(), 0, apply_fn),
-        |(oldstats, blockheight)| {
-            chain_asset_stats_delta(chain, asset_id, oldstats, blockheight + 1, apply_fn)
-        },
-    );
-
-    // save updated stats to cache
-    if let Some(lastblock) = lastblock {
-        chain.store().cache_db().write(
-            vec![asset_cache_row(asset_id, &newstats, &lastblock)],
-            DBFlush::Enable,
-        );
-    }
-
-    newstats
-}
-
-// Update the asset stats with the delta of confirmed txs since start_height
-fn chain_asset_stats_delta<T>(
-    chain: &ChainQuery,
-    asset_id: &AssetId,
-    init_stats: T,
-    start_height: usize,
-    apply_fn: AssetStatApplyFn<T>,
-) -> (T, Option<BlockHash>) {
-    let history_iter = chain
-        .history_iter_scan(b'I', &asset_id.into_inner()[..], start_height)
-        .map(TxHistoryRow::from_row)
-        .filter_map(|history| {
-            chain
-                .tx_confirming_block(&history.get_txid())
-                .map(|blockid| (history, blockid))
-        });
-
-    let mut stats = init_stats;
-    let mut seen_txids = HashSet::new();
-    let mut lastblock = None;
-
-    for (row, blockid) in history_iter {
-        if lastblock != Some(blockid.hash) {
-            seen_txids.clear();
-        }
-        apply_fn(&row.key.txinfo, &mut stats, &mut seen_txids);
-        lastblock = Some(blockid.hash);
-    }
-
-    (stats, lastblock)
-}
-
-// Get mempool asset stats (issued or the pegged asset)
-pub fn mempool_asset_stats<T>(
-    mempool: &Mempool,
-    asset_id: &AssetId,
-    apply_fn: AssetStatApplyFn<T>,
-) -> T
-where
-    T: Default,
-{
-    let mut stats = T::default();
-
-    if let Some(history) = mempool.asset_history.get(asset_id) {
-        let mut seen_txids = HashSet::new();
-        for info in history {
-            apply_fn(info, &mut stats, &mut seen_txids)
-        }
-    }
-
-    stats
-}
-
-fn apply_issued_asset_stats(
-    info: &TxHistoryInfo,
-    stats: &mut IssuedAssetStats,
-    seen_txids: &mut HashSet<Txid>,
-) {
-    if seen_txids.insert(info.get_txid()) {
-        stats.tx_count += 1;
-    }
-
-    match info {
-        TxHistoryInfo::Issuing(issuance) => {
-            stats.issuance_count += 1;
-
-            match issuance.issued_amount {
-                Some(amount) => stats.issued_amount += amount,
-                None => stats.has_blinded_issuances = true,
-            }
-
-            if !issuance.is_reissuance {
-                stats.reissuance_tokens = issuance.token_amount;
-            }
-        }
-
-        TxHistoryInfo::Burning(info) => {
-            stats.burned_amount += info.value;
-        }
-
-        TxHistoryInfo::Funding(_) | TxHistoryInfo::Spending(_) => {
-            // we don't keep funding/spending entries for assets
-            unreachable!();
-        }
-
-        TxHistoryInfo::Pegin(_) | TxHistoryInfo::Pegout(_) => {
-            // issued assets cannot have pegins/pegouts
-            unreachable!();
-        }
-    }
-}
-
-fn apply_pegged_asset_stats(
-    info: &TxHistoryInfo,
-    stats: &mut PeggedAssetStats,
-    seen_txids: &mut HashSet<Txid>,
-) {
-    if seen_txids.insert(info.get_txid()) {
-        stats.tx_count += 1;
-    }
-
-    match info {
-        TxHistoryInfo::Pegin(info) => {
-            stats.peg_in_count += 1;
-            stats.peg_in_amount += info.value;
-        }
-        TxHistoryInfo::Pegout(info) => {
-            stats.peg_out_count += 1;
-            stats.peg_out_amount += info.value;
-        }
-        TxHistoryInfo::Burning(info) => {
-            stats.burn_count += 1;
-            stats.burned_amount += info.value;
-        }
-        TxHistoryInfo::Issuing(_) => {
-            warn!("encountered issuance of native asset, ignoring (possibly freeinitialcoins?)");
-        }
-        TxHistoryInfo::Funding(_) | TxHistoryInfo::Spending(_) => {
-            // these history entries variants are never kept for native assets
-            unreachable!();
-        }
-    }
-}

src/elements/mod.rs

@@ -1,77 +0,0 @@
-use bitcoin::hashes::Hash;
-use elements::hex::ToHex;
-use elements::secp256k1_zkp::ZERO_TWEAK;
-use elements::{confidential::Value, encode::serialize, issuance::ContractHash, AssetId, TxIn};
-
-pub mod asset;
-pub mod peg;
-mod registry;
-
-use asset::get_issuance_entropy;
-pub use asset::{lookup_asset, LiquidAsset};
-pub use registry::{AssetMeta, AssetRegistry, AssetSorting};
-
-#[derive(Serialize, Deserialize, Clone)]
-pub struct IssuanceValue {
-    pub asset_id: String,
-    pub is_reissuance: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub asset_blinding_nonce: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub contract_hash: Option<String>,
-    pub asset_entropy: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub assetamount: Option<u64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub assetamountcommitment: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub tokenamount: Option<u64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub tokenamountcommitment: Option<String>,
-}
-
-impl From<&TxIn> for IssuanceValue {
-    fn from(txin: &TxIn) -> Self {
-        let issuance = &txin.asset_issuance;
-        let is_reissuance = issuance.asset_blinding_nonce != ZERO_TWEAK;
-
-        let asset_entropy = get_issuance_entropy(txin).expect("invalid issuance");
-        let asset_id = AssetId::from_entropy(asset_entropy);
-
-        let contract_hash = if !is_reissuance {
-            Some(ContractHash::from_slice(&issuance.asset_entropy).expect("invalid asset entropy"))
-        } else {
-            None
-        };
-
-        IssuanceValue {
-            asset_id: asset_id.to_hex(),
-            asset_entropy: asset_entropy.to_hex(),
-            contract_hash: contract_hash.map(|h| h.to_hex()),
-            is_reissuance,
-            asset_blinding_nonce: if is_reissuance {
-                Some(hex::encode(issuance.asset_blinding_nonce.as_ref()))
-            } else {
-                None
-            },
-            assetamount: match issuance.amount {
-                Value::Explicit(value) => Some(value),
-                Value::Null => Some(0),
-                Value::Confidential(..) => None,
-            },
-            assetamountcommitment: match issuance.amount {
-                Value::Confidential(..) => Some(hex::encode(serialize(&issuance.amount))),
-                _ => None,
-            },
-            tokenamount: match issuance.inflation_keys {
-                Value::Explicit(value) => Some(value),
-                Value::Null => Some(0),
-                Value::Confidential(..) => None,
-            },
-            tokenamountcommitment: match issuance.inflation_keys {
-                Value::Confidential(..) => Some(hex::encode(serialize(&issuance.inflation_keys))),
-                _ => None,
-            },
-        }
-    }
-}

src/elements/peg.rs

@@ -1,84 +0,0 @@
-use elements::hex::ToHex;
-use elements::{confidential::Asset, PeginData, PegoutData, TxIn, TxOut};
-
-use crate::chain::{bitcoin_genesis_hash, BNetwork, Network};
-use crate::util::FullHash;
-
-pub fn get_pegin_data(txout: &TxIn, network: Network) -> Option<PeginData<'_>> {
-    let pegged_asset_id = network.pegged_asset()?;
-    txout.pegin_data().and_then(|pegin| {
-        if pegin.asset == *pegged_asset_id {
-            Some(pegin)
-        } else {
-            None
-        }
-    })
-}
-
-pub fn get_pegout_data(
-    txout: &TxOut,
-    network: Network,
-    parent_network: BNetwork,
-) -> Option<PegoutData<'_>> {
-    let pegged_asset_id = network.pegged_asset()?;
-    txout.pegout_data().and_then(|pegout| {
-        if pegout.asset == Asset::Explicit(*pegged_asset_id)
-            && pegout.genesis_hash
-                == bitcoin_genesis_hash(match parent_network {
-                    BNetwork::Bitcoin => Network::Liquid,
-                    BNetwork::Testnet | BNetwork::Testnet4 => Network::LiquidTestnet,
-                    BNetwork::Signet => return None,
-                    BNetwork::Regtest => Network::LiquidRegtest,
-                })
-        {
-            Some(pegout)
-        } else {
-            None
-        }
-    })
-}
-
-// API representation of pegout data assocaited with an output
-#[derive(Serialize, Deserialize, Clone)]
-pub struct PegoutValue {
-    pub genesis_hash: String,
-    pub scriptpubkey: bitcoin::ScriptBuf,
-    pub scriptpubkey_asm: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub scriptpubkey_address: Option<String>,
-}
-
-impl PegoutValue {
-    pub fn from_txout(txout: &TxOut, network: Network, parent_network: BNetwork) -> Option<Self> {
-        let pegoutdata = get_pegout_data(txout, network, parent_network)?;
-
-        // pending https://github.com/ElementsProject/rust-elements/pull/69 is merged
-        let scriptpubkey = bitcoin::ScriptBuf::from(pegoutdata.script_pubkey.into_bytes());
-        let address = bitcoin::Address::from_script(&scriptpubkey, parent_network).ok();
-
-        Some(PegoutValue {
-            genesis_hash: pegoutdata.genesis_hash.to_hex(),
-            scriptpubkey_asm: scriptpubkey.to_asm_string(),
-            scriptpubkey_address: address.map(|s| s.to_string()),
-            scriptpubkey,
-        })
-    }
-}
-
-// Inner type for the indexer TxHistoryInfo::Pegin variant
-#[derive(Serialize, Deserialize, Debug)]
-#[cfg_attr(test, derive(PartialEq, Eq))]
-pub struct PeginInfo {
-    pub txid: FullHash,
-    pub vin: u32,
-    pub value: u64,
-}
-
-// Inner type for the indexer TxHistoryInfo::Pegout variant
-#[derive(Serialize, Deserialize, Debug)]
-#[cfg_attr(test, derive(PartialEq, Eq))]
-pub struct PegoutInfo {
-    pub txid: FullHash,
-    pub vout: u32,
-    pub value: u64,
-}

src/elements/registry.rs

@@ -1,298 +0,0 @@
-use std::collections::HashMap;
-use std::str::FromStr;
-use std::sync::{Arc, RwLock};
-use std::time::{Duration, SystemTime};
-use std::{cmp, fs, path, thread};
-
-use serde_json::Value as JsonValue;
-
-use elements::AssetId;
-
-use crate::errors::*;
-
-// length of asset id prefix to use for sub-directory partitioning
-// (in number of hex characters, not bytes)
-
-const DIR_PARTITION_LEN: usize = 2;
-const SEARCH_SORT_CANDIDATE_LIMIT: usize = 2000;
-
-pub struct AssetRegistry {
-    directory: path::PathBuf,
-    assets_cache: HashMap<AssetId, (SystemTime, AssetMeta)>,
-}
-
-pub type AssetEntry<'a> = (&'a AssetId, &'a AssetMeta);
-
-impl AssetRegistry {
-    pub fn new(directory: path::PathBuf) -> Self {
-        Self {
-            directory,
-            assets_cache: Default::default(),
-        }
-    }
-
-    pub fn get(&self, asset_id: &AssetId) -> Option<&AssetMeta> {
-        self.assets_cache
-            .get(asset_id)
-            .map(|(_, metadata)| metadata)
-    }
-
-    pub fn list(
-        &self,
-        start_index: usize,
-        limit: usize,
-        sorting: AssetSorting,
-    ) -> (usize, Vec<AssetEntry<'_>>) {
-        let mut assets: Vec<AssetEntry> = self
-            .assets_cache
-            .iter()
-            .map(|(asset_id, (_, metadata))| (asset_id, metadata))
-            .collect();
-        assets.sort_by(sorting.as_comparator());
-        (
-            assets.len(),
-            assets.into_iter().skip(start_index).take(limit).collect(),
-        )
-    }
-
-    pub fn search(&self, query: &str, limit: usize) -> Vec<AssetEntry<'_>> {
-        let query = query.trim();
-        if query.is_empty() || limit == 0 {
-            return vec![];
-        }
-
-        let (mut results, candidates) = search_by(
-            self.assets_cache
-                .iter()
-                .map(|(asset_id, (_, metadata))| (asset_id, metadata)),
-            query,
-            limit,
-            |metadata| metadata.ticker.as_deref(),
-        );
-
-        if results.len() < limit {
-            let (name_matches, candidates) =
-                search_by(candidates, query, limit - results.len(), |metadata| {
-                    Some(&metadata.name)
-                });
-            results.extend(name_matches);
-
-            if results.len() < limit {
-                let (domain_matches, _) =
-                    search_by(candidates, query, limit - results.len(), AssetMeta::domain);
-                results.extend(domain_matches);
-            }
-        }
-
-        results.truncate(limit);
-        results
-    }
-
-    pub fn fs_sync(&mut self) -> Result<()> {
-        for entry in fs::read_dir(&self.directory).chain_err(|| "failed reading asset dir")? {
-            let entry = entry.chain_err(|| "invalid fh")?;
-            let filetype = entry.file_type().chain_err(|| "failed getting file type")?;
-            if !filetype.is_dir() || entry.file_name().len() != DIR_PARTITION_LEN {
-                continue;
-            }
-
-            for file_entry in
-                fs::read_dir(entry.path()).chain_err(|| "failed reading asset subdir")?
-            {
-                let file_entry = file_entry.chain_err(|| "invalid fh")?;
-                let path = file_entry.path();
-                if path.extension().and_then(|e| e.to_str()) != Some("json") {
-                    continue;
-                }
-
-                let asset_id = AssetId::from_str(
-                    path.file_stem()
-                        .unwrap() // cannot fail if extension() succeeded
-                        .to_str()
-                        .chain_err(|| "invalid filename")?,
-                )
-                .chain_err(|| "invalid filename")?;
-
-                let modified = file_entry
-                    .metadata()
-                    .chain_err(|| "failed reading metadata")?
-                    .modified()
-                    .chain_err(|| "metadata modified failed")?;
-
-                if let Some((last_update, _)) = self.assets_cache.get(&asset_id) {
-                    if *last_update == modified {
-                        continue;
-                    }
-                }
-
-                let metadata: AssetMeta = serde_json::from_str(
-                    &fs::read_to_string(path).chain_err(|| "failed reading file")?,
-                )
-                .chain_err(|| "failed parsing file")?;
-
-                self.assets_cache.insert(asset_id, (modified, metadata));
-            }
-        }
-        Ok(())
-    }
-
-    pub fn spawn_sync(asset_db: Arc<RwLock<AssetRegistry>>) -> thread::JoinHandle<()> {
-        crate::util::spawn_thread("asset-registry", move || loop {
-            if let Err(e) = asset_db.write().unwrap().fs_sync() {
-                error!("registry fs_sync failed: {:?}", e);
-            }
-
-            thread::sleep(Duration::from_secs(15));
-            // TODO handle shutdowm
-        })
-    }
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct AssetMeta {
-    #[serde(skip_serializing_if = "JsonValue::is_null")]
-    pub contract: JsonValue,
-    #[serde(skip_serializing_if = "JsonValue::is_null")]
-    pub entity: JsonValue,
-    pub precision: u8,
-    pub name: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub ticker: Option<String>,
-}
-
-impl AssetMeta {
-    pub(crate) fn domain(&self) -> Option<&str> {
-        self.entity["domain"].as_str()
-    }
-}
-
-pub struct AssetSorting(AssetSortField, AssetSortDir);
-
-pub enum AssetSortField {
-    Name,
-    Domain,
-    Ticker,
-}
-pub enum AssetSortDir {
-    Descending,
-    Ascending,
-}
-
-type Comparator = Box<dyn Fn(&AssetEntry, &AssetEntry) -> cmp::Ordering>;
-
-impl AssetSorting {
-    #[allow(clippy::wrong_self_convention)]
-    fn as_comparator(self) -> Comparator {
-        let sort_fn: Comparator = match self.0 {
-            AssetSortField::Name => {
-                // Order by name first, use asset id as a tie breaker. the other sorting fields
-                // don't require this because they're guaranteed to be unique.
-                Box::new(|a, b| lc_cmp(&a.1.name, &b.1.name).then_with(|| a.0.cmp(b.0)))
-            }
-            AssetSortField::Domain => Box::new(|a, b| a.1.domain().cmp(&b.1.domain())),
-            AssetSortField::Ticker => Box::new(|a, b| lc_cmp_opt(&a.1.ticker, &b.1.ticker)),
-        };
-
-        match self.1 {
-            AssetSortDir::Ascending => sort_fn,
-            AssetSortDir::Descending => Box::new(move |a, b| sort_fn(a, b).reverse()),
-        }
-    }
-
-    pub fn from_query_params(query: &HashMap<String, String>) -> Result<Self> {
-        let field = match query.get("sort_field").map(String::as_str) {
-            None => AssetSortField::Ticker,
-            Some("name") => AssetSortField::Name,
-            Some("domain") => AssetSortField::Domain,
-            Some("ticker") => AssetSortField::Ticker,
-            _ => bail!("invalid sort field"),
-        };
-
-        let dir = match query.get("sort_dir").map(String::as_str) {
-            None => AssetSortDir::Ascending,
-            Some("asc") => AssetSortDir::Ascending,
-            Some("desc") => AssetSortDir::Descending,
-            _ => bail!("invalid sort direction"),
-        };
-
-        Ok(Self(field, dir))
-    }
-}
-
-fn lc_cmp(a: &str, b: &str) -> cmp::Ordering {
-    a.to_lowercase().cmp(&b.to_lowercase())
-}
-fn lc_cmp_opt(a: &Option<String>, b: &Option<String>) -> cmp::Ordering {
-    a.as_ref()
-        .map(|a| a.to_lowercase())
-        .cmp(&b.as_ref().map(|b| b.to_lowercase()))
-}
-
-fn search_by<'a, I, F>(
-    candidates: I,
-    query: &str,
-    limit: usize,
-    field: F,
-) -> (Vec<AssetEntry<'a>>, Vec<AssetEntry<'a>>)
-where
-    I: IntoIterator<Item = AssetEntry<'a>>,
-    F: Fn(&AssetMeta) -> Option<&str>,
-{
-    let mut matches = vec![];
-    let mut remaining = vec![];
-
-    for (asset_id, metadata) in candidates {
-        let position = field(metadata).and_then(|field| {
-            // registry fields are ascii, so we don't need full unicode case-folding
-            ascii_ci_find(field, query).map(|position| (position, field))
-        });
-
-        if let Some((position, field)) = position {
-            if matches.len() >= SEARCH_SORT_CANDIDATE_LIMIT {
-                continue;
-            }
-            matches.push((position, field, asset_id, metadata));
-        } else {
-            remaining.push((asset_id, metadata));
-        }
-    }
-
-    matches.sort_unstable_by(|a, b| {
-        a.0.cmp(&b.0)
-            .then_with(|| ascii_ci_cmp(a.1, b.1))
-            .then_with(|| a.2.cmp(b.2))
-    });
-
-    (
-        matches
-            .into_iter()
-            .take(limit)
-            .map(|(_, _, asset_id, metadata)| (asset_id, metadata))
-            .collect(),
-        remaining,
-    )
-}
-
-// zero-allocation case-insensitive ASCII substring search
-// returns the byte offset of the first match
-fn ascii_ci_find(haystack: &str, needle: &str) -> Option<usize> {
-    let (haystack, needle) = (haystack.as_bytes(), needle.as_bytes());
-    if needle.is_empty() {
-        return Some(0);
-    }
-    haystack
-        .windows(needle.len())
-        .position(|window| window.eq_ignore_ascii_case(needle))
-}
-
-// zero-allocation case-insensitive ASCII string comparison
-fn ascii_ci_cmp(a: &str, b: &str) -> cmp::Ordering {
-    let (a, b) = (a.as_bytes(), b.as_bytes());
-    for i in 0..a.len().min(b.len()) {
-        match a[i].to_ascii_lowercase().cmp(&b[i].to_ascii_lowercase()) {
-            cmp::Ordering::Equal => continue,
-            ord => return ord,
-        }
-    }
-    a.len().cmp(&b.len())
-}

src/errors.rs

@@ -1,4 +1,5 @@
-#![allow(unexpected_cfgs)]
+use chan_signal::Signal;
+
 error_chain! {
     types {
         Error, ErrorKind, ResultExt, Result;
@@ -10,33 +11,9 @@ error_chain! {
             display("Connection error: {}", msg)
         }
 
-        Interrupt(sig: i32) {
+        Interrupt(signal: Signal) {
             description("Interruption by external signal")
-            display("Iterrupted by signal {}", sig)
+            display("Iterrupted by SIG{:?}", signal)
         }
-
-        TooManyUtxos(limit: usize) {
-            description("Too many unspent transaction outputs. Contact support to raise limits.")
-            display("Too many unspent transaction outputs (>{}). Contact support to raise limits.", limit)
-        }
-
-        TooManyTxs(limit: usize) {
-            description("Too many history transactions. Contact support to raise limits.")
-            display("Too many history transactions (>{}). Contact support to raise limits.", limit)
-        }
-
-        #[cfg(feature = "electrum-discovery")]
-        ElectrumClient(e: electrum_client::Error) {
-            description("Electrum client error")
-            display("Electrum client error: {:?}", e)
-        }
-
-    }
-}
-
-#[cfg(feature = "electrum-discovery")]
-impl From<electrum_client::Error> for Error {
-    fn from(e: electrum_client::Error) -> Self {
-        Error::from(ErrorKind::ElectrumClient(e))
     }
 }

src/fake.rs

@@ -0,0 +1,37 @@
+use store::{ReadStore, Row, WriteStore};
+use util::Bytes;
+
+pub struct FakeStore;
+
+impl ReadStore for FakeStore {
+    fn get(&self, _key: &[u8]) -> Option<Bytes> {
+        None
+    }
+    fn scan(&self, _prefix: &[u8]) -> Vec<Row> {
+        vec![]
+    }
+}
+
+impl WriteStore for FakeStore {
+    fn write(&self, _rows: Vec<Row>) {}
+    fn flush(&self) {}
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn test_fakestore() {
+        use fake;
+        use store::{ReadStore, Row, WriteStore};
+
+        let store = fake::FakeStore {};
+        store.write(vec![Row {
+            key: b"k".to_vec(),
+            value: b"v".to_vec(),
+        }]);
+        store.flush();
+        // nothing was actually written
+        assert!(store.get(b"").is_none());
+        assert!(store.scan(b"").is_empty());
+    }
+}

src/index.rs

@@ -0,0 +1,518 @@
+use bincode;
+use bitcoin::consensus::encode::{deserialize, serialize};
+use bitcoin::util::hash::BitcoinHash;
+use bitcoin::util::hash::Sha256dHash;
+use crypto::digest::Digest;
+use crypto::sha2::Sha256;
+use elements::{Block, BlockHeader, Transaction, TxIn, TxOut};
+use std::collections::{HashMap, HashSet};
+use std::iter::FromIterator;
+use std::sync::RwLock;
+
+use daemon::Daemon;
+use metrics::{Counter, Gauge, HistogramOpts, HistogramTimer, HistogramVec, MetricOpts, Metrics};
+use signal::Waiter;
+use store::{ReadStore, Row, WriteStore};
+use util::{
+    full_hash, hash_prefix, spawn_thread, BlockMeta, Bytes, FullHash, HashPrefix, HeaderEntry,
+    HeaderList, HeaderMap, SyncChannel, HASH_PREFIX_LEN,
+};
+
+use config::Config;
+
+use errors::*;
+
+#[derive(Serialize, Deserialize)]
+pub struct TxInKey {
+    pub code: u8,
+    pub prev_hash_prefix: HashPrefix,
+    pub prev_index: u16,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TxInRow {
+    key: TxInKey,
+    pub txid_prefix: HashPrefix,
+}
+
+impl TxInRow {
+    pub fn new(txid: &Sha256dHash, input: &TxIn) -> TxInRow {
+        TxInRow {
+            key: TxInKey {
+                code: b'I',
+                prev_hash_prefix: hash_prefix(&input.previous_output.txid.as_bytes()[..]),
+                prev_index: input.previous_output.vout as u16,
+            },
+            txid_prefix: hash_prefix(&txid[..]),
+        }
+    }
+
+    pub fn filter(txid: &Sha256dHash, output_index: usize) -> Bytes {
+        bincode::serialize(&TxInKey {
+            code: b'I',
+            prev_hash_prefix: hash_prefix(&txid[..]),
+            prev_index: output_index as u16,
+        })
+        .unwrap()
+    }
+
+    pub fn to_row(&self) -> Row {
+        Row {
+            key: bincode::serialize(&self).unwrap(),
+            value: vec![],
+        }
+    }
+
+    pub fn from_row(row: &Row) -> TxInRow {
+        bincode::deserialize(&row.key).expect("failed to parse TxInRow")
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TxOutKey {
+    code: u8,
+    script_hash_prefix: HashPrefix,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TxOutRow {
+    key: TxOutKey,
+    pub txid_prefix: HashPrefix,
+}
+
+impl TxOutRow {
+    pub fn new(txid: &Sha256dHash, output: &TxOut) -> TxOutRow {
+        TxOutRow {
+            key: TxOutKey {
+                code: b'O',
+                script_hash_prefix: hash_prefix(&compute_script_hash(&output.script_pubkey[..])),
+            },
+            txid_prefix: hash_prefix(&txid[..]),
+        }
+    }
+
+    pub fn filter(script_hash: &[u8]) -> Bytes {
+        bincode::serialize(&TxOutKey {
+            code: b'O',
+            script_hash_prefix: hash_prefix(&script_hash[..HASH_PREFIX_LEN]),
+        })
+        .unwrap()
+    }
+
+    pub fn to_row(&self) -> Row {
+        Row {
+            key: bincode::serialize(&self).unwrap(),
+            value: vec![],
+        }
+    }
+
+    pub fn from_row(row: &Row) -> TxOutRow {
+        bincode::deserialize(&row.key).expect("failed to parse TxOutRow")
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct TxKey {
+    code: u8,
+    pub txid: FullHash,
+}
+
+#[derive(Debug)]
+pub struct TxRow {
+    pub key: TxKey,
+    pub height: u32, // value
+    pub blockhash: Sha256dHash,
+}
+
+impl TxRow {
+    pub fn new(txid: &Sha256dHash, height: u32, blockhash: &Sha256dHash) -> TxRow {
+        TxRow {
+            key: TxKey {
+                code: b'T',
+                txid: full_hash(&txid[..]),
+            },
+            height: height,
+            blockhash: blockhash.clone(),
+        }
+    }
+
+    pub fn filter_prefix(txid_prefix: &HashPrefix) -> Bytes {
+        [b"T", &txid_prefix[..]].concat()
+    }
+
+    pub fn filter_full(txid: &Sha256dHash) -> Bytes {
+        [b"T", &txid[..]].concat()
+    }
+
+    pub fn to_row(&self) -> Row {
+        Row {
+            key: bincode::serialize(&self.key).unwrap(),
+            value: bincode::serialize(&(&self.height, &self.blockhash)).unwrap(),
+        }
+    }
+
+    pub fn from_row(row: &Row) -> TxRow {
+        let (height, blockhash): (u32, Sha256dHash) =
+            bincode::deserialize(&row.value).expect("failed to parse tx row");
+        TxRow {
+            key: bincode::deserialize(&row.key).expect("failed to parse TxKey"),
+            height: height,
+            blockhash: blockhash,
+        }
+    }
+}
+
+pub struct RawTxRow {
+    pub key: TxKey,
+    pub rawtx: Bytes,
+}
+
+impl RawTxRow {
+    pub fn new(txid: &Sha256dHash, rawtx: Bytes) -> RawTxRow {
+        RawTxRow {
+            key: TxKey {
+                code: b't',
+                txid: full_hash(&txid[..]),
+            },
+            rawtx: rawtx,
+        }
+    }
+
+    pub fn filter_prefix(txid_prefix: &HashPrefix) -> Bytes {
+        [b"t", &txid_prefix[..]].concat()
+    }
+
+    pub fn filter_full(txid: &Sha256dHash) -> Bytes {
+        [b"t", &txid[..]].concat()
+    }
+
+    pub fn to_row(&self) -> Row {
+        Row {
+            key: bincode::serialize(&self.key).unwrap(),
+            value: bincode::serialize(&self.rawtx).unwrap(),
+        }
+    }
+
+    pub fn from_row(row: &Row) -> RawTxRow {
+        RawTxRow {
+            key: bincode::deserialize(&row.key).expect("failed to parse TxKey for RawTx"),
+            rawtx: bincode::deserialize(&row.value).expect("failed to parse rawtx"),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+struct BlockKey {
+    code: u8,
+    hash: FullHash,
+}
+
+pub fn compute_script_hash(data: &[u8]) -> FullHash {
+    let mut hash = FullHash::default();
+    let mut sha2 = Sha256::new();
+    sha2.input(data);
+    sha2.result(&mut hash);
+    hash
+}
+
+pub fn index_transaction(
+    txn: &Transaction,
+    height: u32,
+    blockhash: &Sha256dHash,
+    rows: &mut Vec<Row>,
+) {
+    let null_hash = Sha256dHash::default();
+    let txid: Sha256dHash = txn.txid();
+    for input in &txn.input {
+        if input.previous_output.txid == null_hash {
+            continue;
+        }
+        rows.push(TxInRow::new(&txid, &input).to_row());
+    }
+    for output in &txn.output {
+        rows.push(TxOutRow::new(&txid, &output).to_row());
+    }
+    // Persist transaction ID and confirmed height/hash
+    rows.push(TxRow::new(&txid, height, blockhash).to_row());
+}
+
+pub fn index_block(block: &Block, height: u32, extended_db_enabled: bool) -> Vec<Row> {
+    let blockhash = block.bitcoin_hash();
+    let mut rows = vec![];
+    for txn in &block.txdata {
+        index_transaction(&txn, height, &blockhash, &mut rows);
+
+        // Persist raw transaction to txstore
+        if extended_db_enabled {
+            rows.push(RawTxRow::new(&txn.txid(), serialize(txn)).to_row()); // @TODO avoid re-serialization
+        }
+    }
+    let blockhash = block.bitcoin_hash();
+    // Persist block hash and header
+    rows.push(Row {
+        key: bincode::serialize(&BlockKey {
+            code: b'B',
+            hash: full_hash(&blockhash[..]),
+        })
+        .unwrap(),
+        value: serialize(&block.header),
+    });
+
+    // Persist block metadata (size, number of txs and sum of txs weight)
+    if extended_db_enabled {
+        let blockmeta = BlockMeta::from(block);
+        rows.push(Row {
+            key: bincode::serialize(&BlockKey {
+                code: b'M',
+                hash: full_hash(&blockhash[..]),
+            })
+            .unwrap(),
+            value: bincode::serialize(&blockmeta).unwrap(),
+        });
+    }
+
+    // Persist list of txids in block
+    if extended_db_enabled {
+        let txids: Vec<Sha256dHash> = block.txdata.iter().map(|tx| tx.txid()).collect();
+        rows.push(Row {
+            key: bincode::serialize(&BlockKey {
+                code: b'X',
+                hash: full_hash(&blockhash[..]),
+            })
+            .unwrap(),
+            value: bincode::serialize(&txids).unwrap(),
+        });
+    }
+
+    rows
+}
+
+pub fn last_indexed_block(blockhash: &Sha256dHash) -> Row {
+    // Store last indexed block (i.e. all previous blocks were indexed)
+    Row {
+        key: b"L".to_vec(),
+        value: serialize(blockhash),
+    }
+}
+
+pub fn read_indexed_blockhashes(store: &ReadStore) -> HashSet<Sha256dHash> {
+    let mut result = HashSet::new();
+    for row in store.scan(b"B") {
+        let key: BlockKey = bincode::deserialize(&row.key).unwrap();
+        result.insert(deserialize(&key.hash).unwrap());
+    }
+    result
+}
+
+fn read_indexed_headers(store: &ReadStore) -> HeaderList {
+    let latest_blockhash: Sha256dHash = match store.get(b"L") {
+        // latest blockheader persisted in the DB.
+        Some(row) => deserialize(&row).unwrap(),
+        None => Sha256dHash::default(),
+    };
+    let mut map = HeaderMap::new();
+    for row in store.scan(b"B") {
+        let key: BlockKey = bincode::deserialize(&row.key).unwrap();
+        let header: BlockHeader = deserialize(&row.value).unwrap();
+        map.insert(deserialize(&key.hash).unwrap(), header);
+    }
+    let mut headers = vec![];
+    let null_hash = Sha256dHash::default();
+    let mut blockhash = latest_blockhash;
+    while blockhash != null_hash {
+        let header = map
+            .remove(&blockhash)
+            .expect(&format!("missing {} header in DB", blockhash));
+        blockhash = header.prev_blockhash;
+        headers.push(header);
+    }
+    headers.reverse();
+    assert_eq!(
+        headers
+            .first()
+            .map(|h| h.prev_blockhash)
+            .unwrap_or(null_hash),
+        null_hash
+    );
+    assert_eq!(
+        headers
+            .last()
+            .map(|h| h.bitcoin_hash())
+            .unwrap_or(null_hash),
+        latest_blockhash
+    );
+    let mut result = HeaderList::empty();
+    let entries = result.order(headers);
+    result.apply(entries);
+    result
+}
+
+struct Stats {
+    blocks: Counter,
+    txns: Counter,
+    vsize: Counter,
+    height: Gauge,
+    duration: HistogramVec,
+}
+
+impl Stats {
+    fn new(metrics: &Metrics) -> Stats {
+        Stats {
+            blocks: metrics.counter(MetricOpts::new("index_blocks", "# of indexed blocks")),
+            txns: metrics.counter(MetricOpts::new("index_txns", "# of indexed transactions")),
+            vsize: metrics.counter(MetricOpts::new("index_vsize", "# of indexed vbytes")),
+            height: metrics.gauge(MetricOpts::new(
+                "index_height",
+                "Last indexed block's height",
+            )),
+            duration: metrics.histogram_vec(
+                HistogramOpts::new("index_duration", "indexing duration (in seconds)"),
+                &["step"],
+            ),
+        }
+    }
+
+    fn update(&self, block: &Block, height: usize) {
+        self.blocks.inc();
+        self.txns.inc_by(block.txdata.len() as i64);
+        for tx in &block.txdata {
+            self.vsize.inc_by(tx.get_weight() as i64 / 4);
+        }
+        self.height.set(height as i64);
+    }
+
+    fn start_timer(&self, step: &str) -> HistogramTimer {
+        self.duration.with_label_values(&[step]).start_timer()
+    }
+}
+
+pub struct Index {
+    // TODO: store also latest snapshot.
+    headers: RwLock<HeaderList>,
+    daemon: Daemon,
+    stats: Stats,
+    batch_size: usize,
+    extended_db_enabled: bool,
+}
+
+impl Index {
+    pub fn load(
+        store: &ReadStore,
+        daemon: &Daemon,
+        metrics: &Metrics,
+        config: &Config,
+    ) -> Result<Index> {
+        let stats = Stats::new(metrics);
+        let headers = read_indexed_headers(store);
+        stats.height.set((headers.len() as i64) - 1);
+        Ok(Index {
+            headers: RwLock::new(headers),
+            daemon: daemon.reconnect()?,
+            stats,
+            batch_size: config.index_batch_size,
+            extended_db_enabled: config.extended_db_enabled,
+        })
+    }
+
+    pub fn reload(&self, store: &ReadStore) {
+        let mut headers = self.headers.write().unwrap();
+        *headers = read_indexed_headers(store);
+    }
+
+    pub fn best_height(&self) -> usize {
+        self.headers.read().unwrap().len() - 1
+    }
+
+    pub fn best_header(&self) -> Option<HeaderEntry> {
+        let headers = self.headers.read().unwrap();
+        headers.header_by_blockhash(headers.tip()).cloned()
+    }
+
+    pub fn best_header_hash(&self) -> Sha256dHash {
+        self.headers.read().unwrap().tip().clone()
+    }
+
+    pub fn get_header(&self, height: usize) -> Option<HeaderEntry> {
+        self.headers
+            .read()
+            .unwrap()
+            .header_by_height(height)
+            .cloned()
+    }
+
+    pub fn get_header_by_hash(&self, hash: &Sha256dHash) -> Option<HeaderEntry> {
+        self.headers
+            .read()
+            .unwrap()
+            .header_by_blockhash(hash)
+            .cloned()
+    }
+
+    pub fn update(&self, store: &WriteStore, waiter: &Waiter) -> Result<Sha256dHash> {
+        let daemon = self.daemon.reconnect()?;
+        let tip = daemon.getbestblockhash()?;
+        let new_headers: Vec<HeaderEntry> = {
+            let indexed_headers = self.headers.read().unwrap();
+            indexed_headers.order(daemon.get_new_headers(&indexed_headers, &tip)?)
+        };
+        new_headers.last().map(|tip| {
+            info!("{:?} ({} left to index)", tip, new_headers.len());
+        });
+        let height_map = HashMap::<Sha256dHash, usize>::from_iter(
+            new_headers.iter().map(|h| (*h.hash(), h.height())),
+        );
+
+        let chan = SyncChannel::new(1);
+        let sender = chan.sender();
+        let blockhashes: Vec<Sha256dHash> = new_headers.iter().map(|h| *h.hash()).collect();
+        let batch_size = self.batch_size;
+        let fetcher = spawn_thread("fetcher", move || {
+            for chunk in blockhashes.chunks(batch_size) {
+                sender
+                    .send(daemon.getblocks(&chunk))
+                    .expect("failed sending blocks to be indexed");
+            }
+            sender
+                .send(Ok(vec![]))
+                .expect("failed sending explicit end of stream");
+        });
+        loop {
+            waiter.poll()?;
+            let timer = self.stats.start_timer("fetch");
+            let batch = chan
+                .receiver()
+                .recv()
+                .expect("block fetch exited prematurely")?;
+            timer.observe_duration();
+            if batch.is_empty() {
+                break;
+            }
+
+            let mut rows = vec![];
+            for block in &batch {
+                let blockhash = block.bitcoin_hash();
+                let height = *height_map
+                    .get(&blockhash)
+                    .expect(&format!("missing header for block {}", blockhash));
+
+                let timer = self.stats.start_timer("index");
+                let mut block_rows = index_block(block, height as u32, self.extended_db_enabled);
+                block_rows.push(last_indexed_block(&blockhash));
+                rows.extend(block_rows);
+                timer.observe_duration();
+                self.stats.update(block, height);
+            }
+            let timer = self.stats.start_timer("write");
+            store.write(rows);
+            timer.observe_duration();
+        }
+        let timer = self.stats.start_timer("flush");
+        store.flush(); // make sure no row is left behind
+        timer.observe_duration();
+
+        fetcher.join().expect("block fetcher failed");
+        self.headers.write().unwrap().apply(new_headers);
+        assert_eq!(tip, *self.headers.read().unwrap().tip());
+        Ok(tip)
+    }
+}

src/lib.rs

@@ -1,7 +1,33 @@
 #![recursion_limit = "1024"]
 
+extern crate base64;
+extern crate bincode;
+extern crate bitcoin;
+extern crate bitcoin_bech32;
+extern crate chan_signal;
+extern crate crypto;
+extern crate dirs;
+extern crate elements;
+extern crate glob;
+extern crate hex;
+extern crate hyper;
+extern crate libc;
+extern crate lru;
+extern crate lru_cache;
+extern crate num_cpus;
+extern crate page_size;
+extern crate prometheus;
 extern crate rocksdb;
+extern crate secp256k1;
+extern crate serde;
+extern crate stderrlog;
+extern crate sysconf;
+extern crate time;
+extern crate tiny_http;
+extern crate url;
 
+#[macro_use]
+extern crate chan;
 #[macro_use]
 extern crate clap;
 #[macro_use]
@@ -15,19 +41,18 @@ extern crate serde_derive;
 #[macro_use]
 extern crate serde_json;
 
-#[macro_use]
-extern crate lazy_static;
-
-pub mod chain;
+pub mod app;
+pub mod bulk;
 pub mod config;
 pub mod daemon;
-pub mod electrum;
 pub mod errors;
+pub mod fake;
+pub mod index;
+pub mod mempool;
 pub mod metrics;
-pub mod new_index;
+pub mod query;
 pub mod rest;
 pub mod signal;
+pub mod store;
 pub mod util;
-
-#[cfg(feature = "liquid")]
-pub mod elements;
+pub mod utils;

src/mempool.rs

@@ -0,0 +1,282 @@
+use bitcoin::util::hash::Sha256dHash;
+use elements::Transaction;
+use hex;
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::iter::FromIterator;
+use std::ops::Bound;
+use std::sync::Mutex;
+
+use daemon::{Daemon, MempoolEntry};
+use index::index_transaction;
+use metrics::{Gauge, GaugeVec, HistogramOpts, HistogramTimer, HistogramVec, MetricOpts, Metrics};
+use store::{ReadStore, Row};
+use util::Bytes;
+
+use errors::*;
+
+const VSIZE_BIN_WIDTH: u32 = 100_000; // in vbytes
+pub const MEMPOOL_HEIGHT: u32 = u32::max_value(); // special "marker" for mempool transactions
+
+struct MempoolStore {
+    map: BTreeMap<Bytes, Vec<Bytes>>,
+}
+
+impl MempoolStore {
+    fn new() -> MempoolStore {
+        MempoolStore {
+            map: BTreeMap::new(),
+        }
+    }
+
+    fn add(&mut self, tx: &Transaction) {
+        let mut rows = vec![];
+        index_transaction(tx, MEMPOOL_HEIGHT, &Sha256dHash::default(), &mut rows);
+        for row in rows {
+            let (key, value) = row.into_pair();
+            self.map.entry(key).or_insert(vec![]).push(value);
+        }
+    }
+
+    fn remove(&mut self, tx: &Transaction) {
+        let mut rows = vec![];
+        index_transaction(tx, MEMPOOL_HEIGHT, &Sha256dHash::default(), &mut rows);
+        for row in rows {
+            let (key, value) = row.into_pair();
+            let no_values_left = {
+                let values = self
+                    .map
+                    .get_mut(&key)
+                    .expect(&format!("missing key {} in mempool", hex::encode(&key)));
+                let last_value = values
+                    .pop()
+                    .expect(&format!("no values found for key {}", hex::encode(&key)));
+                // TxInRow and TxOutRow have an empty value, TxRow has MEMPOOL_HEIGHT as value.
+                assert_eq!(
+                    value,
+                    last_value,
+                    "wrong value for key {}: {}",
+                    hex::encode(&key),
+                    hex::encode(&last_value)
+                );
+                values.is_empty()
+            };
+            if no_values_left {
+                self.map.remove(&key).unwrap();
+            }
+        }
+    }
+}
+
+impl ReadStore for MempoolStore {
+    fn get(&self, key: &[u8]) -> Option<Bytes> {
+        Some(self.map.get(key)?.last()?.to_vec())
+    }
+    fn scan(&self, prefix: &[u8]) -> Vec<Row> {
+        let range = self
+            .map
+            .range((Bound::Included(prefix.to_vec()), Bound::Unbounded));
+        let mut rows = vec![];
+        for (key, values) in range {
+            if !key.starts_with(prefix) {
+                break;
+            }
+            if let Some(value) = values.last() {
+                rows.push(Row {
+                    key: key.to_vec(),
+                    value: value.to_vec(),
+                });
+            }
+        }
+        rows
+    }
+}
+
+struct Item {
+    tx: Transaction,     // stored for faster retrieval and index removal
+    entry: MempoolEntry, // caches mempool fee rates
+}
+
+struct Stats {
+    count: Gauge,
+    update: HistogramVec,
+    vsize: GaugeVec,
+    max_fee_rate: Mutex<f32>,
+}
+
+impl Stats {
+    fn start_timer(&self, step: &str) -> HistogramTimer {
+        self.update.with_label_values(&[step]).start_timer()
+    }
+
+    fn update(&self, entries: &[&MempoolEntry]) {
+        let mut bands: Vec<(f32, u32)> = vec![];
+        let mut fee_rate = 1.0f32; // [sat/vbyte]
+        let mut vsize = 0u32; // vsize of transactions paying <= fee_rate
+        for e in entries {
+            while fee_rate < e.fee_per_vbyte() {
+                bands.push((fee_rate, vsize));
+                fee_rate *= 2.0;
+            }
+            vsize += e.vsize();
+        }
+        let mut max_fee_rate = self.max_fee_rate.lock().unwrap();
+        loop {
+            bands.push((fee_rate, vsize));
+            if fee_rate < *max_fee_rate {
+                fee_rate *= 2.0;
+                continue;
+            }
+            *max_fee_rate = fee_rate;
+            break;
+        }
+        drop(max_fee_rate);
+        for (fee_rate, vsize) in bands {
+            // labels should be ordered by fee_rate value
+            let label = format!("≤{:10.0}", fee_rate);
+            self.vsize.with_label_values(&[&label]).set(vsize as f64);
+        }
+    }
+}
+
+pub struct Tracker {
+    items: HashMap<Sha256dHash, Item>,
+    index: MempoolStore,
+    histogram: Vec<(f32, u32)>,
+    stats: Stats,
+}
+
+impl Tracker {
+    pub fn new(metrics: &Metrics) -> Tracker {
+        Tracker {
+            items: HashMap::new(),
+            index: MempoolStore::new(),
+            histogram: vec![],
+            stats: Stats {
+                count: metrics.gauge(MetricOpts::new(
+                    "mempool_count",
+                    "# of mempool transactions",
+                )),
+                update: metrics.histogram_vec(
+                    HistogramOpts::new("mempool_update", "Time to update mempool (in seconds)"),
+                    &["step"],
+                ),
+                vsize: metrics.gauge_vec(
+                    MetricOpts::new(
+                        "mempool_vsize",
+                        "Total vsize of transactions paying at most given fee rate",
+                    ),
+                    &["fee_rate"],
+                ),
+                max_fee_rate: Mutex::new(1.0),
+            },
+        }
+    }
+
+    pub fn get_txn(&self, txid: &Sha256dHash) -> Option<Transaction> {
+        self.items.get(txid).map(|stats| stats.tx.clone())
+    }
+
+    /// Returns vector of (fee_rate, vsize) pairs, where fee_{n-1} > fee_n and vsize_n is the
+    /// total virtual size of mempool transactions with fee in the bin [fee_{n-1}, fee_n].
+    /// Note: fee_{-1} is implied to be infinite.
+    pub fn fee_histogram(&self) -> &Vec<(f32, u32)> {
+        &self.histogram
+    }
+
+    pub fn index(&self) -> &ReadStore {
+        &self.index
+    }
+
+    pub fn update(&mut self, daemon: &Daemon) -> Result<()> {
+        let timer = self.stats.start_timer("fetch");
+        let new_txids = daemon
+            .getmempooltxids()
+            .chain_err(|| "failed to update mempool from daemon")?;
+        let old_txids = HashSet::from_iter(self.items.keys().cloned());
+        timer.observe_duration();
+
+        let timer = self.stats.start_timer("add");
+        let txids_iter = new_txids.difference(&old_txids);
+        let entries: Vec<(&Sha256dHash, MempoolEntry)> = txids_iter
+            .filter_map(|txid| {
+                match daemon.getmempoolentry(txid) {
+                    Ok(entry) => Some((txid, entry)),
+                    Err(err) => {
+                        warn!("no mempool entry {}: {}", txid, err); // e.g. new block or RBF
+                        None // ignore this transaction for now
+                    }
+                }
+            })
+            .collect();
+        if entries.is_empty() {
+            return Ok(());
+        }
+        let txids: Vec<&Sha256dHash> = entries.iter().map(|(txid, _)| *txid).collect();
+        let txs = match daemon.gettransactions(&txids) {
+            Ok(txs) => txs,
+            Err(err) => {
+                warn!("failed to get transactions {:?}: {}", txids, err); // e.g. new block or RBF
+                return Ok(()); // keep the mempool until next update()
+            }
+        };
+        for ((txid, entry), tx) in entries.into_iter().zip(txs.into_iter()) {
+            assert_eq!(tx.txid(), *txid);
+            self.add(txid, tx, entry);
+        }
+        timer.observe_duration();
+
+        let timer = self.stats.start_timer("remove");
+        for txid in old_txids.difference(&new_txids) {
+            self.remove(txid);
+        }
+        timer.observe_duration();
+
+        let timer = self.stats.start_timer("fees");
+        self.update_fee_histogram();
+        timer.observe_duration();
+
+        self.stats.count.set(self.items.len() as i64);
+        Ok(())
+    }
+
+    fn add(&mut self, txid: &Sha256dHash, tx: Transaction, entry: MempoolEntry) {
+        self.index.add(&tx);
+        self.items.insert(*txid, Item { tx, entry });
+    }
+
+    fn remove(&mut self, txid: &Sha256dHash) {
+        let stats = self
+            .items
+            .remove(txid)
+            .expect(&format!("missing mempool tx {}", txid));
+        self.index.remove(&stats.tx);
+    }
+
+    fn update_fee_histogram(&mut self) {
+        let mut entries: Vec<&MempoolEntry> = self.items.values().map(|stat| &stat.entry).collect();
+        entries.sort_unstable_by(|e1, e2| {
+            e1.fee_per_vbyte().partial_cmp(&e2.fee_per_vbyte()).unwrap()
+        });
+        self.histogram = electrum_fees(&entries);
+        self.stats.update(&entries);
+    }
+}
+
+fn electrum_fees(entries: &[&MempoolEntry]) -> Vec<(f32, u32)> {
+    let mut histogram = vec![];
+    let mut bin_size = 0;
+    let mut last_fee_rate = None;
+    for e in entries.iter().rev() {
+        last_fee_rate = Some(e.fee_per_vbyte());
+        bin_size += e.vsize();
+        if bin_size > VSIZE_BIN_WIDTH {
+            // vsize of transactions paying >= e.fee_per_vbyte()
+            histogram.push((e.fee_per_vbyte(), bin_size));
+            bin_size = 0;
+        }
+    }
+    if let Some(fee_rate) = last_fee_rate {
+        histogram.push((fee_rate, bin_size));
+    }
+    histogram
+}

src/metrics.rs

@@ -5,6 +5,7 @@ use std::io;
 use std::net::SocketAddr;
 use std::thread;
 use std::time::Duration;
+use sysconf;
 use tiny_http;
 
 pub use prometheus::{
@@ -12,9 +13,9 @@ pub use prometheus::{
     IntCounterVec as CounterVec, IntGauge as Gauge, Opts as MetricOpts,
 };
 
-use crate::util::spawn_thread;
+use util::spawn_thread;
 
-use crate::errors::*;
+use errors::*;
 
 pub struct Metrics {
     reg: prometheus::Registry,
@@ -66,9 +67,11 @@ impl Metrics {
     }
 
     pub fn start(&self) {
-        let server = tiny_http::Server::http(self.addr)
-            .unwrap_or_else(|_| panic!("failed to start monitoring HTTP server at {}", self.addr));
-        start_process_exporter(self);
+        let server = tiny_http::Server::http(self.addr).expect(&format!(
+            "failed to start monitoring HTTP server at {}",
+            self.addr
+        ));
+        start_process_exporter(&self);
         let reg = self.reg.clone();
         spawn_thread("metrics", move || loop {
             if let Err(e) = handle_request(&reg, server.recv()) {
@@ -97,14 +100,6 @@ struct Stats {
     fds: usize,
 }
 
-fn get_ticks_per_second() -> Result<f64> {
-    // Safety: This code is taken directly from sysconf
-    match unsafe { libc::sysconf(libc::_SC_CLK_TCK) } {
-        -1 => Err("Clock Tick unsupported".into()),
-        ret => Ok(ret as f64),
-    }
-}
-
 fn parse_stats() -> Result<Stats> {
     if cfg!(target_os = "macos") {
         return Ok(Stats {
@@ -116,14 +111,15 @@ fn parse_stats() -> Result<Stats> {
     let value = fs::read_to_string("/proc/self/stat").chain_err(|| "failed to read stats")?;
     let parts: Vec<&str> = value.split_whitespace().collect();
     let page_size = page_size::get() as u64;
-    let ticks_per_second = get_ticks_per_second().expect("failed to get _SC_CLK_TCK");
+    let ticks_per_second = sysconf::raw::sysconf(sysconf::raw::SysconfVariable::ScClkTck)
+        .expect("failed to get _SC_CLK_TCK") as f64;
 
     let parse_part = |index: usize, name: &str| -> Result<u64> {
-        parts
+        Ok(parts
             .get(index)
             .chain_err(|| format!("missing {}: {:?}", name, parts))?
             .parse::<u64>()
-            .chain_err(|| format!("invalid {}: {:?}", name, parts))
+            .chain_err(|| format!("invalid {}: {:?}", name, parts))?)
     };
 
     // For details, see '/proc/[pid]/stat' section at `man 5 proc`:
@@ -148,7 +144,7 @@ fn start_process_exporter(metrics: &Metrics) {
     spawn_thread("exporter", move || loop {
         match parse_stats() {
             Ok(stats) => {
-                cpu.with_label_values(&["utime"]).set(stats.utime);
+                cpu.with_label_values(&["utime"]).set(stats.utime as f64);
                 rss.set(stats.rss as i64);
                 fds.set(stats.fds as i64);
             }

src/new_index/db.rs

@@ -1,330 +0,0 @@
-use rocksdb;
-
-use std::path::Path;
-
-use crate::config::Config;
-use crate::util::{bincode_util, Bytes};
-
-/// Each version will break any running instance with a DB that has a differing version.
-/// It will also break if light mode is enabled or disabled.
-// 1 = Original DB (since fork from Blockstream)
-// 2 = Add tx position to TxHistory rows and place Spending before Funding
-static DB_VERSION: u32 = 2;
-
-#[derive(Debug, Eq, PartialEq)]
-pub struct DBRow {
-    pub key: Vec<u8>,
-    pub value: Vec<u8>,
-}
-
-pub struct ScanIterator<'a> {
-    prefix: Vec<u8>,
-    iter: rocksdb::DBIterator<'a>,
-    done: bool,
-}
-
-impl Iterator for ScanIterator<'_> {
-    type Item = DBRow;
-
-    fn next(&mut self) -> Option<DBRow> {
-        if self.done {
-            return None;
-        }
-        let (key, value) = self.iter.next().map(Result::ok)??;
-        if !key.starts_with(&self.prefix) {
-            self.done = true;
-            return None;
-        }
-        Some(DBRow {
-            key: key.to_vec(),
-            value: value.to_vec(),
-        })
-    }
-}
-
-pub struct ReverseScanIterator<'a> {
-    prefix: Vec<u8>,
-    iter: rocksdb::DBRawIterator<'a>,
-    done: bool,
-}
-
-impl Iterator for ReverseScanIterator<'_> {
-    type Item = DBRow;
-
-    fn next(&mut self) -> Option<DBRow> {
-        if self.done || !self.iter.valid() {
-            return None;
-        }
-
-        let key = self.iter.key().unwrap();
-        if !key.starts_with(&self.prefix) {
-            self.done = true;
-            return None;
-        }
-
-        let row = DBRow {
-            key: key.into(),
-            value: self.iter.value().unwrap().into(),
-        };
-
-        self.iter.prev();
-
-        Some(row)
-    }
-}
-
-pub struct ReverseScanGroupIterator<'a> {
-    iters: Vec<ReverseScanIterator<'a>>,
-    next_rows: Vec<Option<DBRow>>,
-    value_offset: usize,
-    done: bool,
-}
-
-impl<'a> ReverseScanGroupIterator<'a> {
-    pub fn new(
-        mut iters: Vec<ReverseScanIterator<'a>>,
-        value_offset: usize,
-    ) -> ReverseScanGroupIterator<'a> {
-        let mut next_rows: Vec<Option<DBRow>> = Vec::with_capacity(iters.len());
-        for iter in &mut iters {
-            let next = iter.next();
-            next_rows.push(next);
-        }
-        let done = next_rows.iter().all(|row| row.is_none());
-        ReverseScanGroupIterator {
-            iters,
-            next_rows,
-            value_offset,
-            done,
-        }
-    }
-}
-
-impl Iterator for ReverseScanGroupIterator<'_> {
-    type Item = DBRow;
-
-    fn next(&mut self) -> Option<DBRow> {
-        if self.done {
-            return None;
-        }
-
-        let best_index = self
-            .next_rows
-            .iter()
-            .enumerate()
-            .max_by(|(a_index, a_opt), (b_index, b_opt)| match (a_opt, b_opt) {
-                (None, None) => a_index.cmp(b_index),
-
-                (Some(_), None) => std::cmp::Ordering::Greater,
-
-                (None, Some(_)) => std::cmp::Ordering::Less,
-
-                (Some(a), Some(b)) => a.key[self.value_offset..].cmp(&(b.key[self.value_offset..])),
-            })
-            .map(|(index, _)| index)
-            .unwrap_or(0);
-
-        let best = self.next_rows[best_index].take();
-        self.next_rows[best_index] = self.iters.get_mut(best_index)?.next();
-        if self.next_rows.iter().all(|row| row.is_none()) {
-            self.done = true;
-        }
-
-        best
-    }
-}
-
-#[derive(Debug)]
-pub struct DB {
-    db: rocksdb::DB,
-}
-
-#[derive(Copy, Clone, Debug)]
-pub enum DBFlush {
-    Disable,
-    Enable,
-}
-
-impl DB {
-    pub fn open(path: &Path, config: &Config) -> DB {
-        let db = DB {
-            db: open_raw_db(path, OpenMode::ReadWrite),
-        };
-        db.verify_compatibility(config);
-        db
-    }
-
-    pub fn full_compaction(&self) {
-        // TODO: make sure this doesn't fail silently
-        debug!("starting full compaction on {:?}", self.db);
-        self.db.compact_range(None::<&[u8]>, None::<&[u8]>);
-        debug!("finished full compaction on {:?}", self.db);
-    }
-
-    pub fn enable_auto_compaction(&self) {
-        let opts = [("disable_auto_compactions", "false")];
-        self.db.set_options(&opts).unwrap();
-    }
-
-    pub fn raw_iterator(&self) -> rocksdb::DBRawIterator<'_> {
-        self.db.raw_iterator()
-    }
-
-    pub fn iter_scan(&self, prefix: &[u8]) -> ScanIterator<'_> {
-        ScanIterator {
-            prefix: prefix.to_vec(),
-            iter: self.db.prefix_iterator(prefix),
-            done: false,
-        }
-    }
-
-    pub fn iter_scan_from(&self, prefix: &[u8], start_at: &[u8]) -> ScanIterator<'_> {
-        let iter = self.db.iterator(rocksdb::IteratorMode::From(
-            start_at,
-            rocksdb::Direction::Forward,
-        ));
-        ScanIterator {
-            prefix: prefix.to_vec(),
-            iter,
-            done: false,
-        }
-    }
-
-    pub fn iter_scan_reverse(&self, prefix: &[u8], prefix_max: &[u8]) -> ReverseScanIterator<'_> {
-        let mut iter = self.db.raw_iterator();
-        iter.seek_for_prev(prefix_max);
-
-        ReverseScanIterator {
-            prefix: prefix.to_vec(),
-            iter,
-            done: false,
-        }
-    }
-
-    pub fn iter_scan_group_reverse(
-        &self,
-        prefixes: impl Iterator<Item = (Vec<u8>, Vec<u8>)>,
-        value_offset: usize,
-    ) -> ReverseScanGroupIterator<'_> {
-        let iters = prefixes
-            .map(|(prefix, prefix_max)| {
-                let mut iter = self.db.raw_iterator();
-                iter.seek_for_prev(prefix_max);
-                ReverseScanIterator {
-                    prefix: prefix.to_vec(),
-                    iter,
-                    done: false,
-                }
-            })
-            .collect();
-        ReverseScanGroupIterator::new(iters, value_offset)
-    }
-
-    pub fn write(&self, mut rows: Vec<DBRow>, flush: DBFlush) {
-        debug!(
-            "writing {} rows to {:?}, flush={:?}",
-            rows.len(),
-            self.db,
-            flush
-        );
-        rows.sort_unstable_by(|a, b| a.key.cmp(&b.key));
-        let mut batch = rocksdb::WriteBatch::default();
-        for row in rows {
-            batch.put(&row.key, &row.value);
-        }
-        let do_flush = match flush {
-            DBFlush::Enable => true,
-            DBFlush::Disable => false,
-        };
-        let mut opts = rocksdb::WriteOptions::new();
-        opts.set_sync(do_flush);
-        opts.disable_wal(!do_flush);
-        self.db.write_opt(batch, &opts).unwrap();
-    }
-
-    pub fn delete(&self, keys: Vec<Vec<u8>>) {
-        debug!("deleting {} rows from {:?}", keys.len(), self.db);
-        for key in keys {
-            let _ = self.db.delete(key).inspect_err(|err| {
-                warn!("Error while deleting DB row: {err}");
-            });
-        }
-    }
-
-    pub fn flush(&self) {
-        self.db.flush().unwrap();
-    }
-
-    pub fn put(&self, key: &[u8], value: &[u8]) {
-        self.db.put(key, value).unwrap();
-    }
-
-    pub fn put_sync(&self, key: &[u8], value: &[u8]) {
-        let mut opts = rocksdb::WriteOptions::new();
-        opts.set_sync(true);
-        self.db.put_opt(key, value, &opts).unwrap();
-    }
-
-    pub fn get(&self, key: &[u8]) -> Option<Bytes> {
-        self.db.get(key).unwrap().map(|v| v.to_vec())
-    }
-
-    fn verify_compatibility(&self, config: &Config) {
-        let mut compatibility_bytes = bincode_util::serialize_little(&DB_VERSION).unwrap();
-
-        if config.light_mode {
-            // append a byte to indicate light_mode is enabled.
-            // we're not letting bincode serialize this so that the compatiblity bytes won't change
-            // (and require a reindex) when light_mode is disabled. this should be chagned the next
-            // time we bump DB_VERSION and require a re-index anyway.
-            compatibility_bytes.push(1);
-        }
-
-        match self.get(b"V") {
-            None => self.put(b"V", &compatibility_bytes),
-            Some(ref x) if x != &compatibility_bytes => {
-                panic!("Incompatible database found. Please reindex.")
-            }
-            Some(_) => (),
-        }
-    }
-}
-
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-#[repr(u8)]
-pub enum OpenMode {
-    ReadOnly,
-    ReadWrite,
-}
-
-pub fn open_raw_db<T: rocksdb::ThreadMode>(
-    path: &Path,
-    read_mode: OpenMode,
-) -> rocksdb::DBWithThreadMode<T> {
-    debug!("opening DB at {:?}", path);
-    let mut db_opts = rocksdb::Options::default();
-    db_opts.create_if_missing(true);
-    db_opts.set_max_open_files(100_000); // TODO: make sure to `ulimit -n` this process correctly
-    db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
-    db_opts.set_compression_type(rocksdb::DBCompressionType::None);
-    db_opts.set_target_file_size_base(1_073_741_824);
-    db_opts.set_write_buffer_size(256 << 20);
-    db_opts.set_disable_auto_compactions(true); // for initial bulk load
-
-    // db_opts.set_advise_random_on_open(???);
-    db_opts.set_compaction_readahead_size(1 << 20);
-    db_opts.increase_parallelism(2);
-
-    // let mut block_opts = rocksdb::BlockBasedOptions::default();
-    // block_opts.set_block_size(???);
-
-    match read_mode {
-        OpenMode::ReadOnly => {
-            rocksdb::DBWithThreadMode::<T>::open_for_read_only(&db_opts, path, false)
-                .expect("failed to open RocksDB (READ ONLY)")
-        }
-        OpenMode::ReadWrite => {
-            rocksdb::DBWithThreadMode::<T>::open(&db_opts, path).expect("failed to open RocksDB")
-        }
-    }
-}

src/new_index/fetch.rs

@@ -1,299 +0,0 @@
-use rayon::prelude::*;
-
-#[cfg(not(feature = "liquid"))]
-use bitcoin::consensus::encode::{deserialize, Decodable};
-#[cfg(feature = "liquid")]
-use elements::encode::{deserialize, Decodable};
-
-use std::collections::HashMap;
-use std::fs;
-use std::io::Cursor;
-use std::path::PathBuf;
-use std::thread;
-
-use crate::chain::{Block, BlockHash, BlockSizeCompat};
-use crate::daemon::Daemon;
-use crate::errors::*;
-use crate::util::{spawn_thread, HeaderEntry, SyncChannel};
-
-#[derive(Clone, Copy, Debug)]
-pub enum FetchFrom {
-    Bitcoind,
-    BlkFiles,
-}
-
-pub fn start_fetcher(
-    from: FetchFrom,
-    daemon: &Daemon,
-    new_headers: Vec<HeaderEntry>,
-) -> Result<Fetcher<Vec<BlockEntry>>> {
-    let fetcher = match from {
-        FetchFrom::Bitcoind => bitcoind_fetcher,
-        FetchFrom::BlkFiles => blkfiles_fetcher,
-    };
-    fetcher(daemon, new_headers)
-}
-
-pub struct BlockEntry {
-    pub block: Block,
-    pub entry: HeaderEntry,
-    pub size: u32,
-}
-
-type SizedBlock = (Block, u32);
-
-pub struct SequentialFetcher<T> {
-    fetcher: Box<dyn FnOnce() -> Vec<Vec<T>>>,
-}
-
-impl<T> SequentialFetcher<T> {
-    fn from<F: FnOnce() -> Vec<Vec<T>> + 'static>(pre_func: F) -> Self {
-        SequentialFetcher {
-            fetcher: Box::new(pre_func),
-        }
-    }
-
-    pub fn map<FN>(self, mut func: FN)
-    where
-        FN: FnMut(Vec<T>),
-    {
-        for item in (self.fetcher)() {
-            func(item);
-        }
-    }
-}
-
-pub fn bitcoind_sequential_fetcher(
-    daemon: &Daemon,
-    new_headers: Vec<HeaderEntry>,
-) -> Result<SequentialFetcher<BlockEntry>> {
-    let daemon = daemon.reconnect()?;
-    Ok(SequentialFetcher::from(move || {
-        new_headers
-            .chunks(100)
-            .map(|entries| {
-                let blockhashes: Vec<BlockHash> = entries.iter().map(|e| *e.hash()).collect();
-                let blocks = daemon
-                    .getblocks(&blockhashes)
-                    .expect("failed to get blocks from bitcoind");
-                assert_eq!(blocks.len(), entries.len());
-                let block_entries: Vec<BlockEntry> = blocks
-                    .into_iter()
-                    .zip(entries)
-                    .map(|(block, entry)| BlockEntry {
-                        entry: entry.clone(), // TODO: remove this clone()
-                        size: block.get_block_size() as u32,
-                        block,
-                    })
-                    .collect();
-                assert_eq!(block_entries.len(), entries.len());
-                block_entries
-            })
-            .collect()
-    }))
-}
-
-pub struct Fetcher<T> {
-    receiver: crossbeam_channel::Receiver<T>,
-    thread: thread::JoinHandle<()>,
-}
-
-impl<T> Fetcher<T> {
-    fn from(receiver: crossbeam_channel::Receiver<T>, thread: thread::JoinHandle<()>) -> Self {
-        Fetcher { receiver, thread }
-    }
-
-    pub fn map<F>(self, mut func: F)
-    where
-        F: FnMut(T),
-    {
-        for item in self.receiver {
-            func(item);
-        }
-        self.thread.join().expect("fetcher thread panicked")
-    }
-}
-
-fn bitcoind_fetcher(
-    daemon: &Daemon,
-    new_headers: Vec<HeaderEntry>,
-) -> Result<Fetcher<Vec<BlockEntry>>> {
-    if let Some(tip) = new_headers.last() {
-        debug!("{:?} ({} left to index)", tip, new_headers.len());
-    };
-    let daemon = daemon.reconnect()?;
-    let chan = SyncChannel::new(1);
-    let sender = chan.sender();
-    Ok(Fetcher::from(
-        chan.into_receiver(),
-        spawn_thread("bitcoind_fetcher", move || {
-            for entries in new_headers.chunks(100) {
-                let blockhashes: Vec<BlockHash> = entries.iter().map(|e| *e.hash()).collect();
-                let blocks = daemon
-                    .getblocks(&blockhashes)
-                    .expect("failed to get blocks from bitcoind");
-                assert_eq!(blocks.len(), entries.len());
-                let block_entries: Vec<BlockEntry> = blocks
-                    .into_iter()
-                    .zip(entries)
-                    .map(|(block, entry)| BlockEntry {
-                        entry: entry.clone(), // TODO: remove this clone()
-                        size: block.get_block_size() as u32,
-                        block,
-                    })
-                    .collect();
-                assert_eq!(block_entries.len(), entries.len());
-                sender
-                    .send(block_entries)
-                    .expect("failed to send fetched blocks");
-            }
-        }),
-    ))
-}
-
-fn blkfiles_fetcher(
-    daemon: &Daemon,
-    new_headers: Vec<HeaderEntry>,
-) -> Result<Fetcher<Vec<BlockEntry>>> {
-    let magic = daemon.magic();
-    let blk_files = daemon.list_blk_files()?;
-
-    let chan = SyncChannel::new(1);
-    let sender = chan.sender();
-
-    let mut entry_map: HashMap<BlockHash, HeaderEntry> =
-        new_headers.into_iter().map(|h| (*h.hash(), h)).collect();
-
-    let parser = blkfiles_parser(blkfiles_reader(blk_files), magic);
-    Ok(Fetcher::from(
-        chan.into_receiver(),
-        spawn_thread("blkfiles_fetcher", move || {
-            parser.map(|sizedblocks| {
-                let block_entries: Vec<BlockEntry> = sizedblocks
-                    .into_iter()
-                    .filter_map(|(block, size)| {
-                        let blockhash = block.block_hash();
-                        entry_map
-                            .remove(&blockhash)
-                            .map(|entry| BlockEntry { block, entry, size })
-                            .or_else(|| {
-                                trace!("skipping block {}", blockhash);
-                                None
-                            })
-                    })
-                    .collect();
-                trace!("fetched {} blocks", block_entries.len());
-                sender
-                    .send(block_entries)
-                    .expect("failed to send blocks entries from blk*.dat files");
-            });
-            if !entry_map.is_empty() {
-                panic!(
-                    "failed to index {} blocks from blk*.dat files",
-                    entry_map.len()
-                )
-            }
-        }),
-    ))
-}
-
-fn blkfiles_reader(blk_files: Vec<PathBuf>) -> Fetcher<Vec<u8>> {
-    let chan = SyncChannel::new(1);
-    let sender = chan.sender();
-    let xor_key = blk_files.first().and_then(|p| {
-        let xor_file = p
-            .parent()
-            .expect("blk.dat files must exist in a directory")
-            .join("xor.dat");
-        if xor_file.exists() {
-            Some(fs::read(xor_file).expect("xor.dat exists"))
-        } else {
-            None
-        }
-    });
-
-    Fetcher::from(
-        chan.into_receiver(),
-        spawn_thread("blkfiles_reader", move || {
-            for path in blk_files {
-                trace!("reading {:?}", path);
-                let mut blob = fs::read(&path)
-                    .unwrap_or_else(|e| panic!("failed to read {:?}: {:?}", path, e));
-
-                // If the xor.dat exists. Use it to decrypt the block files.
-                if let Some(xor_key) = &xor_key {
-                    for (&key, byte) in xor_key.iter().cycle().zip(blob.iter_mut()) {
-                        *byte ^= key;
-                    }
-                }
-
-                sender
-                    .send(blob)
-                    .unwrap_or_else(|_| panic!("failed to send {:?} contents", path));
-            }
-        }),
-    )
-}
-
-fn blkfiles_parser(blobs: Fetcher<Vec<u8>>, magic: u32) -> Fetcher<Vec<SizedBlock>> {
-    let chan = SyncChannel::new(1);
-    let sender = chan.sender();
-
-    Fetcher::from(
-        chan.into_receiver(),
-        spawn_thread("blkfiles_parser", move || {
-            blobs.map(|blob| {
-                trace!("parsing {} bytes", blob.len());
-                let blocks = parse_blocks(blob, magic).expect("failed to parse blk*.dat file");
-                sender
-                    .send(blocks)
-                    .expect("failed to send blocks from blk*.dat file");
-            });
-        }),
-    )
-}
-
-fn parse_blocks(blob: Vec<u8>, magic: u32) -> Result<Vec<SizedBlock>> {
-    let mut cursor = Cursor::new(&blob);
-    let mut slices = vec![];
-    let max_pos = blob.len() as u64;
-
-    while cursor.position() < max_pos {
-        let offset = cursor.position();
-        match u32::consensus_decode(&mut cursor) {
-            Ok(value) => {
-                if magic != value {
-                    cursor.set_position(offset + 1);
-                    continue;
-                }
-            }
-            Err(_) => break, // EOF
-        };
-        let block_size = u32::consensus_decode(&mut cursor).chain_err(|| "no block size")?;
-        let start = cursor.position();
-        let end = start + block_size as u64;
-
-        // If Core's WriteBlockToDisk ftell fails, only the magic bytes and size will be written
-        // and the block body won't be written to the blk*.dat file.
-        // Since the first 4 bytes should contain the block's version, we can skip such blocks
-        // by peeking the cursor (and skipping previous `magic` and `block_size`).
-        match u32::consensus_decode(&mut cursor) {
-            Ok(value) => {
-                if magic == value {
-                    cursor.set_position(start);
-                    continue;
-                }
-            }
-            Err(_) => break, // EOF
-        }
-        slices.push((&blob[start as usize..end as usize], block_size));
-        cursor.set_position(end);
-    }
-
-    Ok(super::THREAD_POOL.install(|| {
-        slices
-            .into_par_iter()
-            .map(|(slice, size)| (deserialize(slice).expect("failed to parse Block"), size))
-            .collect()
-    }))
-}

src/new_index/mempool.rs

@@ -1,741 +0,0 @@
-use bounded_vec_deque::BoundedVecDeque;
-use itertools::Itertools;
-
-#[cfg(not(feature = "liquid"))]
-use bitcoin::consensus::encode::serialize;
-#[cfg(feature = "liquid")]
-use elements::{encode::serialize, AssetId};
-
-use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
-use std::iter::FromIterator;
-use std::ops::Bound::{Excluded, Unbounded};
-use std::sync::{Arc, RwLock};
-use std::time::{Duration, Instant};
-
-use crate::chain::{deserialize, Network, OutPoint, Transaction, TxOut, Txid, TxidCompat};
-use crate::config::Config;
-use crate::daemon::Daemon;
-use crate::errors::*;
-use crate::metrics::{GaugeVec, HistogramOpts, HistogramVec, MetricOpts, Metrics};
-use crate::new_index::{
-    compute_script_hash, schema::FullHash, ChainQuery, FundingInfo, ScriptStats, SpendingInfo,
-    SpendingInput, TxHistoryInfo, Utxo,
-};
-use crate::util::fees::{make_fee_histogram, TxFeeInfo};
-use crate::util::{extract_tx_prevouts, full_hash, has_prevout, is_spendable, Bytes};
-
-#[cfg(feature = "liquid")]
-use crate::elements::asset;
-
-pub struct Mempool {
-    chain: Arc<ChainQuery>,
-    config: Arc<Config>,
-    txstore: BTreeMap<Txid, Transaction>,
-    feeinfo: HashMap<Txid, TxFeeInfo>,
-    history: HashMap<FullHash, Vec<TxHistoryInfo>>, // ScriptHash -> {history_entries}
-    edges: HashMap<OutPoint, (Txid, u32)>,          // OutPoint -> (spending_txid, spending_vin)
-    recent: BoundedVecDeque<TxOverview>,            // The N most recent txs to enter the mempool
-    backlog_stats: (BacklogStats, Instant),
-
-    // monitoring
-    latency: HistogramVec, // mempool requests latency
-    delta: HistogramVec,   // # of added/removed txs
-    count: GaugeVec,       // current state of the mempool
-
-    // elements only
-    #[cfg(feature = "liquid")]
-    pub asset_history: HashMap<AssetId, Vec<TxHistoryInfo>>,
-    #[cfg(feature = "liquid")]
-    pub asset_issuance: HashMap<AssetId, asset::AssetRow>,
-}
-
-// A simplified transaction view used for the list of most recent transactions
-#[derive(Serialize)]
-pub struct TxOverview {
-    txid: Txid,
-    fee: u64,
-    vsize: u32,
-    #[cfg(not(feature = "liquid"))]
-    value: u64,
-}
-
-impl Mempool {
-    pub fn new(chain: Arc<ChainQuery>, metrics: &Metrics, config: Arc<Config>) -> Self {
-        Mempool {
-            chain,
-            txstore: BTreeMap::new(),
-            feeinfo: HashMap::new(),
-            history: HashMap::new(),
-            edges: HashMap::new(),
-            recent: BoundedVecDeque::new(config.mempool_recent_txs_size),
-            backlog_stats: (
-                BacklogStats::default(),
-                Instant::now() - Duration::from_secs(config.mempool_backlog_stats_ttl),
-            ),
-            latency: metrics.histogram_vec(
-                HistogramOpts::new("mempool_latency", "Mempool requests latency (in seconds)"),
-                &["part"],
-            ),
-            delta: metrics.histogram_vec(
-                HistogramOpts::new("mempool_delta", "# of transactions added/removed"),
-                &["type"],
-            ),
-            count: metrics.gauge_vec(
-                MetricOpts::new("mempool_count", "# of elements currently at the mempool"),
-                &["type"],
-            ),
-
-            #[cfg(feature = "liquid")]
-            asset_history: HashMap::new(),
-            #[cfg(feature = "liquid")]
-            asset_issuance: HashMap::new(),
-            config,
-        }
-    }
-
-    pub fn network(&self) -> Network {
-        self.config.network_type
-    }
-
-    pub fn lookup_txn(&self, txid: &Txid) -> Option<Transaction> {
-        self.txstore.get(txid).cloned()
-    }
-
-    pub fn lookup_raw_txn(&self, txid: &Txid) -> Option<Bytes> {
-        self.txstore.get(txid).map(serialize)
-    }
-
-    pub fn lookup_spend(&self, outpoint: &OutPoint) -> Option<SpendingInput> {
-        self.edges.get(outpoint).map(|(txid, vin)| SpendingInput {
-            txid: *txid,
-            vin: *vin,
-            confirmed: None,
-        })
-    }
-
-    pub fn has_spend(&self, outpoint: &OutPoint) -> bool {
-        self.edges.contains_key(outpoint)
-    }
-
-    pub fn get_tx_fee(&self, txid: &Txid) -> Option<u64> {
-        Some(self.feeinfo.get(txid)?.fee)
-    }
-
-    pub fn has_unconfirmed_parents(&self, txid: &Txid) -> bool {
-        let tx = match self.txstore.get(txid) {
-            Some(tx) => tx,
-            None => return false,
-        };
-        tx.input
-            .iter()
-            .any(|txin| self.txstore.contains_key(&txin.previous_output.txid))
-    }
-
-    pub fn history(
-        &self,
-        scripthash: &[u8],
-        last_seen_txid: Option<&Txid>,
-        limit: usize,
-    ) -> Vec<Transaction> {
-        let _timer = self.latency.with_label_values(&["history"]).start_timer();
-        self.history
-            .get(scripthash)
-            .map_or_else(std::vec::Vec::new, |entries| {
-                self._history(entries, last_seen_txid, limit)
-            })
-    }
-
-    pub fn history_txids_iter<'a>(&'a self, scripthash: &[u8]) -> impl Iterator<Item = Txid> + 'a {
-        self.history
-            .get(scripthash)
-            .into_iter()
-            .flat_map(|v| v.iter().map(|e| e.get_txid()).unique())
-    }
-
-    fn _history(
-        &self,
-        entries: &[TxHistoryInfo],
-        last_seen_txid: Option<&Txid>,
-        limit: usize,
-    ) -> Vec<Transaction> {
-        entries
-            .iter()
-            .map(|e| e.get_txid())
-            .unique()
-            // TODO seek directly to last seen tx without reading earlier rows
-            .skip_while(|txid| {
-                // skip until we reach the last_seen_txid
-                last_seen_txid.is_some_and(|last_seen_txid| last_seen_txid != txid)
-            })
-            .skip(match last_seen_txid {
-                Some(_) => 1, // skip the last_seen_txid itself
-                None => 0,
-            })
-            .take(limit)
-            .map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
-            .cloned()
-            .collect()
-    }
-
-    pub fn history_group(
-        &self,
-        scripthashes: &[[u8; 32]],
-        last_seen_txid: Option<&Txid>,
-        limit: usize,
-    ) -> Vec<Transaction> {
-        let _timer = self
-            .latency
-            .with_label_values(&["history_group"])
-            .start_timer();
-        scripthashes
-            .iter()
-            .filter_map(|scripthash| self.history.get(&scripthash[..]))
-            .flat_map(|entries| entries.iter())
-            .map(|e| e.get_txid())
-            .unique()
-            // TODO seek directly to last seen tx without reading earlier rows
-            .skip_while(|txid| {
-                // skip until we reach the last_seen_txid
-                last_seen_txid.is_some_and(|last_seen_txid| last_seen_txid != txid)
-            })
-            .skip(match last_seen_txid {
-                Some(_) => 1, // skip the last_seen_txid itself
-                None => 0,
-            })
-            .take(limit)
-            .map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
-            .cloned()
-            .collect()
-    }
-
-    pub fn history_txids_iter_group<'a>(
-        &'a self,
-        scripthashes: &'a [[u8; 32]],
-    ) -> impl Iterator<Item = Txid> + 'a {
-        scripthashes
-            .iter()
-            .filter_map(move |scripthash| self.history.get(&scripthash[..]))
-            .flat_map(|entries| entries.iter())
-            .map(|entry| entry.get_txid())
-            .unique()
-    }
-
-    pub fn history_txids(&self, scripthash: &[u8], limit: usize) -> Vec<Txid> {
-        let _timer = self
-            .latency
-            .with_label_values(&["history_txids"])
-            .start_timer();
-        match self.history.get(scripthash) {
-            None => vec![],
-            Some(entries) => entries
-                .iter()
-                .map(|e| e.get_txid())
-                .unique()
-                .take(limit)
-                .collect(),
-        }
-    }
-
-    pub fn utxo(&self, scripthash: &[u8]) -> Vec<Utxo> {
-        let _timer = self.latency.with_label_values(&["utxo"]).start_timer();
-        let entries = match self.history.get(scripthash) {
-            None => return vec![],
-            Some(entries) => entries,
-        };
-
-        entries
-            .iter()
-            .filter_map(|entry| match entry {
-                TxHistoryInfo::Funding(info) => {
-                    // Liquid requires some additional information from the txo that's not available in the TxHistoryInfo index.
-                    #[cfg(feature = "liquid")]
-                    let txo = self.lookup_txo(&entry.get_funded_outpoint())?;
-
-                    Some(Utxo {
-                        txid: deserialize(&info.txid).expect("invalid txid"),
-                        vout: info.vout,
-                        value: info.value,
-                        confirmed: None,
-
-                        #[cfg(feature = "liquid")]
-                        asset: txo.asset,
-                        #[cfg(feature = "liquid")]
-                        nonce: txo.nonce,
-                        #[cfg(feature = "liquid")]
-                        witness: txo.witness,
-                    })
-                }
-                TxHistoryInfo::Spending(_) => None,
-                #[cfg(feature = "liquid")]
-                TxHistoryInfo::Issuing(_)
-                | TxHistoryInfo::Burning(_)
-                | TxHistoryInfo::Pegin(_)
-                | TxHistoryInfo::Pegout(_) => unreachable!(),
-            })
-            .filter(|utxo| !self.has_spend(&OutPoint::from(utxo)))
-            .collect()
-    }
-
-    // @XXX avoid code duplication with ChainQuery::stats()?
-    pub fn stats(&self, scripthash: &[u8]) -> ScriptStats {
-        let _timer = self.latency.with_label_values(&["stats"]).start_timer();
-        let mut stats = ScriptStats::default();
-        let mut seen_txids = HashSet::new();
-
-        let entries = match self.history.get(scripthash) {
-            None => return stats,
-            Some(entries) => entries,
-        };
-
-        for entry in entries {
-            if seen_txids.insert(entry.get_txid()) {
-                stats.tx_count += 1;
-            }
-
-            match entry {
-                #[cfg(not(feature = "liquid"))]
-                TxHistoryInfo::Funding(info) => {
-                    stats.funded_txo_count += 1;
-                    stats.funded_txo_sum += info.value;
-                }
-
-                #[cfg(not(feature = "liquid"))]
-                TxHistoryInfo::Spending(info) => {
-                    stats.spent_txo_count += 1;
-                    stats.spent_txo_sum += info.value;
-                }
-
-                // Elements
-                #[cfg(feature = "liquid")]
-                TxHistoryInfo::Funding(_) => {
-                    stats.funded_txo_count += 1;
-                }
-                #[cfg(feature = "liquid")]
-                TxHistoryInfo::Spending(_) => {
-                    stats.spent_txo_count += 1;
-                }
-                #[cfg(feature = "liquid")]
-                TxHistoryInfo::Issuing(_)
-                | TxHistoryInfo::Burning(_)
-                | TxHistoryInfo::Pegin(_)
-                | TxHistoryInfo::Pegout(_) => unreachable!(),
-            };
-        }
-
-        stats
-    }
-
-    // Get all txids in the mempool
-    pub fn txids(&self) -> Vec<&Txid> {
-        let _timer = self.latency.with_label_values(&["txids"]).start_timer();
-        self.txstore.keys().collect()
-    }
-
-    // Get n txids after the given txid in the mempool
-    pub fn txids_page(&self, n: usize, start: Option<Txid>) -> Vec<&Txid> {
-        let _timer = self
-            .latency
-            .with_label_values(&["txids_page"])
-            .start_timer();
-        let start_bound = match start {
-            Some(txid) => Excluded(txid),
-            None => Unbounded,
-        };
-
-        self.txstore
-            .range((start_bound, Unbounded))
-            .take(n)
-            .map(|(k, _v)| k)
-            .collect()
-    }
-
-    // Get all txs in the mempool
-    pub fn txs(&self) -> Vec<Transaction> {
-        let _timer = self.latency.with_label_values(&["txs"]).start_timer();
-        self.txstore.values().cloned().collect()
-    }
-
-    // Get n txs after the given txid in the mempool
-    pub fn txs_page(&self, n: usize, start: Option<Txid>) -> Vec<Transaction> {
-        let _timer = self.latency.with_label_values(&["txs_page"]).start_timer();
-        let mut page = Vec::with_capacity(n);
-        let start_bound = match start {
-            Some(txid) => Excluded(txid),
-            None => Unbounded,
-        };
-
-        self.txstore
-            .range((start_bound, Unbounded))
-            .take(n)
-            .for_each(|(_, value)| {
-                page.push(value.clone());
-            });
-
-        page
-    }
-
-    // Get an overview of the most recent transactions
-    pub fn recent_txs_overview(&self) -> Vec<&TxOverview> {
-        // We don't bother ever deleting elements from the recent list.
-        // It may contain outdated txs that are no longer in the mempool,
-        // until they get pushed out by newer transactions.
-        self.recent.iter().collect()
-    }
-
-    pub fn backlog_stats(&self) -> &BacklogStats {
-        &self.backlog_stats.0
-    }
-
-    pub fn unique_txids(&self) -> HashSet<Txid> {
-        HashSet::from_iter(self.txstore.keys().cloned())
-    }
-
-    pub fn update(mempool: &RwLock<Mempool>, daemon: &Daemon) -> Result<()> {
-        // 1. Start the metrics timer and get the current mempool txids
-        // [LOCK] Takes read lock for whole scope.
-        let (_timer, old_txids) = {
-            let mempool = mempool.read().unwrap();
-            (
-                mempool.latency.with_label_values(&["update"]).start_timer(),
-                mempool.unique_txids(),
-            )
-        };
-
-        // 2. Get all the mempool txids from the RPC.
-        // [LOCK] No lock taken. Wait for RPC request. Get lists of remove/add txes.
-        let all_txids = daemon
-            .getmempooltxids()
-            .chain_err(|| "failed to update mempool from daemon")?;
-        let txids_to_remove: HashSet<&Txid> = old_txids.difference(&all_txids).collect();
-        let txids_to_add: Vec<&Txid> = all_txids.difference(&old_txids).collect();
-
-        // 3. Remove missing transactions. Even if we are unable to download new transactions from
-        // the daemon, we still want to remove the transactions that are no longer in the mempool.
-        // [LOCK] Write lock is released at the end of the call to remove().
-        mempool.write().unwrap().remove(txids_to_remove);
-
-        // 4. Download the new transactions from the daemon's mempool
-        // [LOCK] No lock taken, waiting for RPC response.
-        let txs_to_add = daemon
-            .gettransactions(&txids_to_add)
-            .chain_err(|| format!("failed to get {} transactions", txids_to_add.len()))?;
-
-        // 4. Update local mempool to match daemon's state
-        // [LOCK] Takes Write lock for whole scope.
-        {
-            let mut mempool = mempool.write().unwrap();
-            // Add new transactions
-            if txs_to_add.len() > mempool.add(txs_to_add) {
-                debug!("Mempool update added less transactions than expected");
-            }
-
-            mempool
-                .count
-                .with_label_values(&["txs"])
-                .set(mempool.txstore.len() as f64);
-
-            // Update cached backlog stats (if expired)
-            if mempool.backlog_stats.1.elapsed()
-                > Duration::from_secs(mempool.config.mempool_backlog_stats_ttl)
-            {
-                let _timer = mempool
-                    .latency
-                    .with_label_values(&["update_backlog_stats"])
-                    .start_timer();
-                mempool.backlog_stats = (BacklogStats::new(&mempool.feeinfo), Instant::now());
-            }
-
-            Ok(())
-        }
-    }
-
-    pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Txid) -> Result<()> {
-        if !self.txstore.contains_key(txid) {
-            if let Ok(tx) = daemon.getmempooltx(txid) {
-                if self.add(vec![tx]) == 0 {
-                    return Err(format!(
-                        "Unable to add {txid} to mempool likely due to missing parents."
-                    )
-                    .into());
-                }
-            }
-        }
-        Ok(())
-    }
-
-    /// Add transactions to the mempool.
-    ///
-    /// The return value is the number of transactions processed.
-    #[must_use = "Must deal with [[input vec's length]] > [[result]]."]
-    fn add(&mut self, txs: Vec<Transaction>) -> usize {
-        self.delta
-            .with_label_values(&["add"])
-            .observe(txs.len() as f64);
-        let _timer = self.latency.with_label_values(&["add"]).start_timer();
-        let txlen = txs.len();
-        if txlen == 0 {
-            return 0;
-        }
-        debug!("Adding {} transactions to Mempool", txlen);
-
-        let mut txids = Vec::with_capacity(txs.len());
-        // Phase 1: add to txstore
-        for tx in txs {
-            let txid = tx.get_txid();
-            // Only push if it doesn't already exist.
-            // This is important now that update doesn't lock during
-            // the entire function body.
-            if self.txstore.insert(txid, tx).is_none() {
-                txids.push(txid);
-            }
-        }
-
-        // Phase 2: index history and spend edges (some txos can be missing)
-        let txos = self.lookup_txos(&self.get_prevouts(&txids));
-
-        // Count how many transactions were actually processed.
-        let mut processed_count = 0;
-
-        // Phase 3: Iterate over the transactions and do the following:
-        // 1. Find all of the TxOuts of each input parent using `txos`
-        // 2. If any parent wasn't found, skip parsing this transaction
-        // 3. Insert TxFeeInfo into info.
-        // 4. Push TxOverview into recent tx queue.
-        // 5. Create the Spend and Fund TxHistory structs for inputs + outputs
-        // 6. Insert all TxHistory into history.
-        // 7. Insert the tx edges into edges (HashMap of (Outpoint, (Txid, vin)))
-        // 8. (Liquid only) Parse assets of tx.
-        for txid in txids {
-            let tx = self.txstore.get(&txid).expect("missing tx from txstore");
-
-            let prevouts = match extract_tx_prevouts(tx, &txos) {
-                Ok(v) => v,
-                Err(e) => {
-                    warn!("Skipping tx {txid} missing parent error: {e}");
-                    continue;
-                }
-            };
-            let txid_bytes = full_hash(&txid[..]);
-
-            // Get feeinfo for caching and recent tx overview
-            let feeinfo = TxFeeInfo::new(tx, &prevouts, self.config.network_type);
-
-            // recent is an BoundedVecDeque that automatically evicts the oldest elements
-            self.recent.push_front(TxOverview {
-                txid,
-                fee: feeinfo.fee,
-                vsize: feeinfo.vsize,
-                #[cfg(not(feature = "liquid"))]
-                value: prevouts
-                    .values()
-                    .map(|prevout| prevout.value.to_sat())
-                    .sum(),
-            });
-
-            self.feeinfo.insert(txid, feeinfo);
-
-            // An iterator over (ScriptHash, TxHistoryInfo)
-            let spending = prevouts.into_iter().map(|(input_index, prevout)| {
-                let txi = tx.input.get(input_index as usize).unwrap();
-                #[cfg(not(feature = "liquid"))]
-                let value = prevout.value.to_sat();
-                #[cfg(feature = "liquid")]
-                let value = prevout.value;
-                (
-                    compute_script_hash(&prevout.script_pubkey),
-                    TxHistoryInfo::Spending(SpendingInfo {
-                        txid: txid_bytes,
-                        vin: input_index,
-                        prev_txid: full_hash(&txi.previous_output.txid[..]),
-                        prev_vout: txi.previous_output.vout,
-                        value,
-                    }),
-                )
-            });
-
-            let config = &self.config;
-
-            // An iterator over (ScriptHash, TxHistoryInfo)
-            let funding = tx
-                .output
-                .iter()
-                .enumerate()
-                .filter(|(_, txo)| is_spendable(txo) || config.index_unspendables)
-                .map(|(index, txo)| {
-                    #[cfg(not(feature = "liquid"))]
-                    let value = txo.value.to_sat();
-                    #[cfg(feature = "liquid")]
-                    let value = txo.value;
-                    (
-                        compute_script_hash(&txo.script_pubkey),
-                        TxHistoryInfo::Funding(FundingInfo {
-                            txid: txid_bytes,
-                            vout: index as u32,
-                            value,
-                        }),
-                    )
-                });
-
-            // Index funding/spending history entries and spend edges
-            for (scripthash, entry) in funding.chain(spending) {
-                self.history.entry(scripthash).or_default().push(entry);
-            }
-            for (i, txi) in tx.input.iter().enumerate() {
-                self.edges.insert(txi.previous_output, (txid, i as u32));
-            }
-
-            // Index issued assets & native asset pegins/pegouts/burns
-            #[cfg(feature = "liquid")]
-            asset::index_mempool_tx_assets(
-                tx,
-                self.config.network_type,
-                self.config.parent_network,
-                &mut self.asset_history,
-                &mut self.asset_issuance,
-            );
-
-            processed_count += 1;
-        }
-
-        processed_count
-    }
-
-    /// Returns None if the lookup fails (mempool transaction RBF-ed etc.)
-    pub fn lookup_txo(&self, outpoint: &OutPoint) -> Option<TxOut> {
-        let mut outpoints = BTreeSet::new();
-        outpoints.insert(*outpoint);
-        // This can possibly be None now
-        self.lookup_txos(&outpoints).remove(outpoint)
-    }
-
-    /// For a given set of OutPoints, return a HashMap<OutPoint, TxOut>
-    ///
-    /// Not all OutPoints from mempool transactions are guaranteed to be there.
-    /// Ensure you deal with the None case in your logic.
-    pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
-        let _timer = self
-            .latency
-            .with_label_values(&["lookup_txos"])
-            .start_timer();
-
-        let confirmed_txos = self.chain.lookup_avail_txos(outpoints);
-
-        let mempool_txos = outpoints
-            .iter()
-            .filter(|outpoint| !confirmed_txos.contains_key(outpoint))
-            .flat_map(|outpoint| {
-                self.txstore
-                    .get(&outpoint.txid)
-                    .and_then(|tx| tx.output.get(outpoint.vout as usize).cloned())
-                    .map(|txout| (*outpoint, txout))
-                    .or_else(|| {
-                        warn!("missing outpoint {:?}", outpoint);
-                        None
-                    })
-            })
-            .collect::<HashMap<OutPoint, TxOut>>();
-
-        let mut txos = confirmed_txos;
-        txos.extend(mempool_txos);
-        txos
-    }
-
-    fn get_prevouts(&self, txids: &[Txid]) -> BTreeSet<OutPoint> {
-        let _timer = self
-            .latency
-            .with_label_values(&["get_prevouts"])
-            .start_timer();
-
-        txids
-            .iter()
-            .map(|txid| self.txstore.get(txid).expect("missing mempool tx"))
-            .flat_map(|tx| {
-                tx.input
-                    .iter()
-                    .filter(|txin| has_prevout(txin))
-                    .map(|txin| txin.previous_output)
-            })
-            .collect()
-    }
-
-    fn remove(&mut self, to_remove: HashSet<&Txid>) {
-        self.delta
-            .with_label_values(&["remove"])
-            .observe(to_remove.len() as f64);
-        let _timer = self.latency.with_label_values(&["remove"]).start_timer();
-
-        for txid in &to_remove {
-            self.txstore
-                .remove(*txid)
-                .unwrap_or_else(|| panic!("missing mempool tx {}", txid));
-
-            self.feeinfo.remove(*txid).or_else(|| {
-                warn!("missing mempool tx feeinfo {}", txid);
-                None
-            });
-        }
-
-        // TODO: make it more efficient (currently it takes O(|mempool|) time)
-        self.history.retain(|_scripthash, entries| {
-            entries.retain(|entry| !to_remove.contains(&entry.get_txid()));
-            !entries.is_empty()
-        });
-
-        #[cfg(feature = "liquid")]
-        asset::remove_mempool_tx_assets(
-            &to_remove,
-            &mut self.asset_history,
-            &mut self.asset_issuance,
-        );
-
-        self.edges
-            .retain(|_outpoint, (txid, _vin)| !to_remove.contains(txid));
-    }
-
-    #[cfg(feature = "liquid")]
-    pub fn asset_history(&self, asset_id: &AssetId, limit: usize) -> Vec<Transaction> {
-        let _timer = self
-            .latency
-            .with_label_values(&["asset_history"])
-            .start_timer();
-        self.asset_history
-            .get(asset_id)
-            .map_or_else(std::vec::Vec::new, |entries| {
-                self._history(entries, None, limit)
-            })
-    }
-}
-
-#[derive(Serialize)]
-pub struct BacklogStats {
-    pub count: u32,
-    pub vsize: u32,     // in virtual bytes (= weight/4)
-    pub total_fee: u64, // in satoshis
-    pub fee_histogram: Vec<(f32, u32)>,
-}
-
-impl BacklogStats {
-    fn default() -> Self {
-        BacklogStats {
-            count: 0,
-            vsize: 0,
-            total_fee: 0,
-            fee_histogram: vec![(0.0, 0)],
-        }
-    }
-
-    fn new(feeinfo: &HashMap<Txid, TxFeeInfo>) -> Self {
-        let (count, vsize, total_fee) = feeinfo
-            .values()
-            .fold((0, 0, 0), |(count, vsize, fee), feeinfo| {
-                (count + 1, vsize + feeinfo.vsize, fee + feeinfo.fee)
-            });
-
-        BacklogStats {
-            count,
-            vsize,
-            total_fee,
-            fee_histogram: make_fee_histogram(feeinfo.values().collect()),
-        }
-    }
-}

src/new_index/mod.rs

@@ -1,25 +0,0 @@
-pub mod db;
-mod fetch;
-mod mempool;
-pub mod precache;
-mod query;
-pub mod schema;
-
-use std::sync::LazyLock;
-
-pub(crate) static THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
-    rayon::ThreadPoolBuilder::new()
-        .num_threads(0) // 0 = use number of logical CPUs
-        .thread_name(|i| format!("electrs-worker-{}", i))
-        .build()
-        .expect("failed to create global rayon thread pool")
-});
-
-pub use self::db::{DBRow, DB};
-pub use self::fetch::{BlockEntry, FetchFrom};
-pub use self::mempool::Mempool;
-pub use self::query::Query;
-pub use self::schema::{
-    compute_script_hash, parse_hash, ChainQuery, FundingInfo, Indexer, ScriptStats, SpendingInfo,
-    SpendingInput, Store, TxHistoryInfo, TxHistoryKey, TxHistoryRow, Utxo,
-};

src/new_index/precache.rs

@@ -1,70 +0,0 @@
-use crate::errors::*;
-use crate::new_index::ChainQuery;
-use crate::util::{full_hash, FullHash};
-
-use rayon::prelude::*;
-
-use hex;
-use std::fs::File;
-use std::io;
-use std::io::prelude::*;
-use std::sync::{atomic::AtomicUsize, Arc};
-use std::time::Instant;
-
-pub fn precache(chain: Arc<ChainQuery>, scripthashes: Vec<FullHash>, threads: usize) {
-    let total = scripthashes.len();
-    info!(
-        "Pre-caching stats and utxo set on {} threads for {} scripthashes",
-        threads, total
-    );
-
-    let pool = rayon::ThreadPoolBuilder::new()
-        .num_threads(threads)
-        .thread_name(|i| format!("precache-{}", i))
-        .build()
-        .unwrap();
-    let now = Instant::now();
-    let counter = AtomicUsize::new(0);
-    std::thread::spawn(move || {
-        pool.install(|| {
-            scripthashes
-                .par_iter()
-                .for_each(|scripthash| {
-                    // First, cache
-                    chain.stats(&scripthash[..], crate::new_index::db::DBFlush::Disable);
-                    let _ = chain.utxo(&scripthash[..], usize::MAX, crate::new_index::db::DBFlush::Disable);
-
-                    // Then, increment the counter
-                    let pre_increment = counter.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
-                    let post_increment_counter = pre_increment + 1;
-
-                    // Then, log
-                    if post_increment_counter % 500 == 0 {
-                        let now_millis = now.elapsed().as_millis();
-                        info!("{post_increment_counter}/{total} Processed in {now_millis} ms running pre-cache for scripthash");
-                    }
-
-                    // Every 10k counts, flush the DB to disk
-                    if post_increment_counter % 10000 == 0 {
-                        info!("Flushing cache_db... {post_increment_counter}");
-                        chain.store().cache_db().flush();
-                        info!("Done Flushing cache_db!!! {post_increment_counter}");
-                    }
-                })
-        });
-        // After everything is done, flush the cache
-        chain.store().cache_db().flush();
-    });
-}
-
-pub fn scripthashes_from_file(path: String) -> Result<Vec<FullHash>> {
-    let reader =
-        io::BufReader::new(File::open(path).chain_err(|| "cannot open precache scripthash file")?);
-    reader
-        .lines()
-        .map(|line| {
-            let line = line.chain_err(|| "cannot read scripthash line")?;
-            Ok(full_hash(&hex::decode(line).chain_err(|| "invalid hex")?))
-        })
-        .collect()
-}

src/new_index/query.rs

@@ -1,333 +0,0 @@
-use rayon::prelude::*;
-
-use std::collections::{BTreeSet, HashMap};
-use std::sync::{Arc, RwLock, RwLockReadGuard};
-use std::time::{Duration, Instant};
-
-use crate::chain::{Network, OutPoint, Transaction, TxOut, Txid, TxidCompat};
-use crate::config::Config;
-use crate::daemon::{Daemon, MempoolAcceptResult, SubmitPackageResult};
-use crate::errors::*;
-use crate::new_index::{ChainQuery, Mempool, ScriptStats, SpendingInput, Utxo};
-use crate::util::{is_spendable, BlockId, Bytes, TransactionStatus};
-
-#[cfg(feature = "liquid")]
-use crate::{
-    chain::{asset::AssetRegistryLock, AssetId},
-    elements::{lookup_asset, AssetMeta, AssetRegistry, AssetSorting, LiquidAsset},
-};
-
-const FEE_ESTIMATES_TTL: u64 = 60; // seconds
-
-const CONF_TARGETS: [u16; 28] = [
-    1u16, 2u16, 3u16, 4u16, 5u16, 6u16, 7u16, 8u16, 9u16, 10u16, 11u16, 12u16, 13u16, 14u16, 15u16,
-    16u16, 17u16, 18u16, 19u16, 20u16, 21u16, 22u16, 23u16, 24u16, 25u16, 144u16, 504u16, 1008u16,
-];
-
-pub struct Query {
-    chain: Arc<ChainQuery>, // TODO: should be used as read-only
-    mempool: Arc<RwLock<Mempool>>,
-    daemon: Arc<Daemon>,
-    config: Arc<Config>,
-    cached_estimates: RwLock<(HashMap<u16, f64>, Option<Instant>)>,
-    cached_relayfee: RwLock<Option<f64>>,
-    #[cfg(feature = "liquid")]
-    asset_db: Option<Arc<RwLock<AssetRegistry>>>,
-}
-
-impl Query {
-    #[cfg(not(feature = "liquid"))]
-    pub fn new(
-        chain: Arc<ChainQuery>,
-        mempool: Arc<RwLock<Mempool>>,
-        daemon: Arc<Daemon>,
-        config: Arc<Config>,
-    ) -> Self {
-        Query {
-            chain,
-            mempool,
-            daemon,
-            config,
-            cached_estimates: RwLock::new((HashMap::new(), None)),
-            cached_relayfee: RwLock::new(None),
-        }
-    }
-
-    pub fn chain(&self) -> &ChainQuery {
-        &self.chain
-    }
-
-    pub fn config(&self) -> &Config {
-        &self.config
-    }
-
-    pub fn network(&self) -> Network {
-        self.config.network_type
-    }
-
-    pub fn mempool(&self) -> RwLockReadGuard<'_, Mempool> {
-        self.mempool.read().unwrap()
-    }
-
-    pub fn broadcast_raw(&self, txhex: &str) -> Result<Txid> {
-        let txid = self.daemon.broadcast_raw(txhex)?;
-        // The important part is whether we succeeded in broadcasting.
-        // Ignore errors in adding to the cache and show an internal warning.
-        if let Err(e) = self
-            .mempool
-            .write()
-            .unwrap()
-            .add_by_txid(&self.daemon, &txid)
-        {
-            warn!(
-                "broadcast_raw of {txid} succeeded to broadcast \
-                but failed to add to mempool-electrs Mempool cache: {e}"
-            );
-        }
-        Ok(txid)
-    }
-
-    pub fn test_mempool_accept(
-        &self,
-        txhex: Vec<String>,
-        maxfeerate: Option<f64>,
-    ) -> Result<Vec<MempoolAcceptResult>> {
-        self.daemon.test_mempool_accept(txhex, maxfeerate)
-    }
-
-    pub fn submit_package(
-        &self,
-        txhex: Vec<String>,
-        maxfeerate: Option<f64>,
-        maxburnamount: Option<f64>,
-    ) -> Result<SubmitPackageResult> {
-        self.daemon.submit_package(txhex, maxfeerate, maxburnamount)
-    }
-
-    pub fn utxo(&self, scripthash: &[u8]) -> Result<Vec<Utxo>> {
-        let mut utxos = self.chain.utxo(
-            scripthash,
-            self.config.utxos_limit,
-            super::db::DBFlush::Enable,
-        )?;
-        let mempool = self.mempool();
-        utxos.retain(|utxo| !mempool.has_spend(&OutPoint::from(utxo)));
-        utxos.extend(mempool.utxo(scripthash));
-        Ok(utxos)
-    }
-
-    pub fn history_txids(&self, scripthash: &[u8], limit: usize) -> Vec<(Txid, Option<BlockId>)> {
-        let confirmed_txids = self.chain.history_txids(scripthash, limit);
-        let confirmed_len = confirmed_txids.len();
-        let confirmed_txids = confirmed_txids.into_iter().map(|(tx, b)| (tx, Some(b)));
-
-        let mempool_txids = self
-            .mempool()
-            .history_txids(scripthash, limit - confirmed_len)
-            .into_iter()
-            .map(|tx| (tx, None));
-
-        confirmed_txids.chain(mempool_txids).collect()
-    }
-
-    pub fn stats(&self, scripthash: &[u8]) -> (ScriptStats, ScriptStats) {
-        (
-            self.chain.stats(scripthash, super::db::DBFlush::Enable),
-            self.mempool().stats(scripthash),
-        )
-    }
-
-    pub fn lookup_txn(&self, txid: &Txid) -> Option<Transaction> {
-        self.chain
-            .lookup_txn(txid, None)
-            .or_else(|| self.mempool().lookup_txn(txid))
-    }
-    pub fn lookup_raw_txn(&self, txid: &Txid) -> Option<Bytes> {
-        self.chain
-            .lookup_raw_txn(txid, None)
-            .or_else(|| self.mempool().lookup_raw_txn(txid))
-    }
-
-    /// Not all OutPoints from mempool transactions are guaranteed to be included in the result
-    pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
-        // the mempool lookup_txos() internally looks up confirmed txos as well
-        self.mempool().lookup_txos(outpoints)
-    }
-
-    pub fn lookup_spend(&self, outpoint: &OutPoint) -> Option<SpendingInput> {
-        self.chain
-            .lookup_spend(outpoint)
-            .or_else(|| self.mempool().lookup_spend(outpoint))
-    }
-
-    pub fn lookup_tx_spends(&self, tx: Transaction) -> Vec<Option<SpendingInput>> {
-        let txid = tx.get_txid();
-
-        tx.output
-            .par_iter()
-            .enumerate()
-            .map(|(vout, txout)| {
-                if is_spendable(txout) {
-                    self.lookup_spend(&OutPoint {
-                        txid,
-                        vout: vout as u32,
-                    })
-                } else {
-                    None
-                }
-            })
-            .collect()
-    }
-
-    pub fn get_tx_status(&self, txid: &Txid) -> TransactionStatus {
-        TransactionStatus::from(self.chain.tx_confirming_block(txid))
-    }
-
-    pub fn get_mempool_tx_fee(&self, txid: &Txid) -> Option<u64> {
-        self.mempool().get_tx_fee(txid)
-    }
-
-    pub fn has_unconfirmed_parents(&self, txid: &Txid) -> bool {
-        self.mempool().has_unconfirmed_parents(txid)
-    }
-
-    pub fn estimate_fee(&self, conf_target: u16) -> Option<f64> {
-        if self.config.network_type.is_regtest() {
-            return self.get_relayfee().ok();
-        }
-        if let (ref cache, Some(cache_time)) = *self.cached_estimates.read().unwrap() {
-            if cache_time.elapsed() < Duration::from_secs(FEE_ESTIMATES_TTL) {
-                return cache.get(&conf_target).copied();
-            }
-        }
-
-        self.update_fee_estimates();
-        self.cached_estimates
-            .read()
-            .unwrap()
-            .0
-            .get(&conf_target)
-            .copied()
-    }
-
-    pub fn estimate_fee_map(&self) -> HashMap<u16, f64> {
-        if let (ref cache, Some(cache_time)) = *self.cached_estimates.read().unwrap() {
-            if cache_time.elapsed() < Duration::from_secs(FEE_ESTIMATES_TTL) {
-                return cache.clone();
-            }
-        }
-
-        self.update_fee_estimates();
-        self.cached_estimates.read().unwrap().0.clone()
-    }
-
-    fn update_fee_estimates(&self) {
-        match self.daemon.estimatesmartfee_batch(&CONF_TARGETS) {
-            Ok(estimates) => {
-                *self.cached_estimates.write().unwrap() = (estimates, Some(Instant::now()));
-            }
-            Err(err) => {
-                warn!("failed estimating feerates: {:?}", err);
-            }
-        }
-    }
-
-    pub fn get_relayfee(&self) -> Result<f64> {
-        if let Some(cached) = *self.cached_relayfee.read().unwrap() {
-            return Ok(cached);
-        }
-
-        let relayfee = self.daemon.get_relayfee()?;
-        self.cached_relayfee.write().unwrap().replace(relayfee);
-        Ok(relayfee)
-    }
-
-    #[cfg(feature = "liquid")]
-    pub fn new(
-        chain: Arc<ChainQuery>,
-        mempool: Arc<RwLock<Mempool>>,
-        daemon: Arc<Daemon>,
-        config: Arc<Config>,
-        asset_db: Option<Arc<RwLock<AssetRegistry>>>,
-    ) -> Self {
-        Query {
-            chain,
-            mempool,
-            daemon,
-            config,
-            asset_db,
-            cached_estimates: RwLock::new((HashMap::new(), None)),
-            cached_relayfee: RwLock::new(None),
-        }
-    }
-
-    #[cfg(feature = "liquid")]
-    pub fn lookup_asset(&self, asset_id: &AssetId) -> Result<Option<LiquidAsset>> {
-        lookup_asset(
-            self,
-            self.asset_db.as_ref().map(AssetRegistryLock::RwLock),
-            asset_id,
-            None,
-        )
-    }
-
-    #[cfg(feature = "liquid")]
-    pub fn lookup_registry_asset(&self, asset_id: &AssetId) -> Result<Option<AssetMeta>> {
-        let asset_db = self
-            .asset_db
-            .as_ref()
-            .chain_err(|| "asset registry unavailable")?;
-        Ok(asset_db.read().unwrap().get(asset_id).cloned())
-    }
-
-    #[cfg(feature = "liquid")]
-    pub fn list_registry_assets(
-        &self,
-        start_index: usize,
-        limit: usize,
-        sorting: AssetSorting,
-    ) -> Result<(usize, Vec<LiquidAsset>)> {
-        let asset_db = match &self.asset_db {
-            None => return Ok((0, vec![])),
-            Some(db) => db.read().unwrap(),
-        };
-        let (total_num, results) = asset_db.list(start_index, limit, sorting);
-        // Attach on-chain information alongside the registry metadata
-        let results = results
-            .into_iter()
-            .map(|(asset_id, metadata)| {
-                lookup_asset(
-                    self,
-                    Some(AssetRegistryLock::RwLockReadGuard(&asset_db)),
-                    asset_id,
-                    Some(metadata),
-                )?
-                .chain_err(|| "missing registered asset")
-            })
-            .collect::<Result<Vec<_>>>()?;
-        Ok((total_num, results))
-    }
-
-    #[cfg(feature = "liquid")]
-    pub fn search_registry_assets<T, F>(
-        &self,
-        search: &str,
-        limit: usize,
-        mut map: F,
-    ) -> Result<Vec<T>>
-    where
-        F: FnMut(&AssetId, &AssetMeta) -> T,
-    {
-        let asset_db = self
-            .asset_db
-            .as_ref()
-            .chain_err(|| "asset registry unavailable")?;
-        Ok(asset_db
-            .read()
-            .unwrap()
-            .search(search, limit)
-            .into_iter()
-            .map(|(asset_id, metadata)| map(asset_id, metadata))
-            .collect())
-    }
-}

src/notify.rs

@@ -0,0 +1,70 @@
+// TODO: network::socket::Socket needs to be reimplemented.
+
+use bitcoin::network::constants::Network;
+use bitcoin::network::message::NetworkMessage;
+use bitcoin::network::message_blockdata::InvType;
+use bitcoin::network::socket::Socket;
+use bitcoin::util::hash::Sha256dHash;
+use bitcoin::util::Error;
+
+use std::sync::mpsc::Sender;
+use std::thread;
+use std::time::Duration;
+
+use util;
+
+fn connect() -> Result<Socket, Error> {
+    let mut sock = Socket::new(Network::Bitcoin);
+    sock.connect("127.0.0.1", 8333)?;
+    Ok(sock)
+}
+
+fn handle(mut sock: Socket, tx: Sender<Sha256dHash>) {
+    let mut outgoing = vec![sock.version_message(0).unwrap()];
+    loop {
+        for msg in outgoing.split_off(0) {
+            trace!("send {:?}", msg);
+            if let Err(e) = sock.send_message(msg.clone()) {
+                warn!("failed to connect to node: {}", e);
+                break;
+            }
+        }
+        // Receive new message
+        let msg = match sock.receive_message() {
+            Ok(msg) => msg,
+            Err(e) => {
+                warn!("failed to receive p2p message: {}", e);
+                break;
+            }
+        };
+        trace!("recv {:?}", msg);
+        match msg {
+            NetworkMessage::Alert(_) => continue, // deprecated
+            NetworkMessage::Version(_) => outgoing.push(NetworkMessage::Verack),
+            NetworkMessage::Ping(nonce) => outgoing.push(NetworkMessage::Pong(nonce)),
+            NetworkMessage::Inv(ref inventory) => {
+                inventory
+                    .iter()
+                    .filter(|inv| inv.inv_type == InvType::Block)
+                    .for_each(|inv| tx.send(inv.hash).expect("failed to send message"));
+            }
+            _ => (),
+        };
+    }
+}
+
+pub fn run() -> util::Channel<Sha256dHash> {
+    let chan = util::Channel::new();
+    let tx = chan.sender();
+
+    util::spawn_thread("p2p", move || loop {
+        // TODO: support testnet and regtest as well.
+        match connect() {
+            Ok(sock) => handle(sock, tx.clone()),
+            Err(e) => warn!("p2p error: {}", e),
+        }
+        thread::sleep(Duration::from_secs(3));
+    });
+
+    chan
+}

src/query.rs

@@ -0,0 +1,672 @@
+use bincode;
+use bitcoin::consensus::encode::{deserialize, serialize};
+use bitcoin::util::hash::Sha256dHash;
+use elements::{confidential, Block, Transaction};
+use std::collections::{BTreeMap, HashMap};
+use std::sync::{Arc, RwLock};
+
+use app::App;
+use index::{compute_script_hash, RawTxRow, TxInRow, TxOutRow, TxRow};
+use mempool::Tracker;
+use metrics::{HistogramOpts, HistogramVec, Metrics};
+use serde_json::Value;
+use store::{ReadStore, Row};
+use util::{
+    BlockHeaderMeta, BlockMeta, BlockStatus, Bytes, HashPrefix, HeaderEntry, TransactionStatus,
+};
+
+use errors::*;
+
+const FUNDING_TXN_LIMIT: usize = 100;
+
+#[derive(Clone)]
+pub struct FundingOutput {
+    pub txn: Option<TxnHeight>,
+    pub txn_id: Sha256dHash,
+    pub height: u32,
+    pub output_index: usize,
+    pub value: u64,
+    pub asset: Option<Sha256dHash>,
+}
+
+impl From<OutPoint> for FundingOutput {
+    fn from(out: OutPoint) -> Self {
+        FundingOutput {
+            txn_id: out.0,
+            output_index: out.1,
+            txn: None,
+            height: 0,
+            value: 0,
+            asset: None,
+        }
+    }
+}
+type OutPoint = (Sha256dHash, usize); // (txid, output_index)
+
+pub struct SpendingInput {
+    pub txn: Option<TxnHeight>,
+    pub txn_id: Sha256dHash,
+    pub height: u32,
+    pub input_index: usize,
+    pub funding_output: OutPoint,
+    pub value: u64,
+}
+
+pub struct Status {
+    confirmed: (Vec<FundingOutput>, Vec<SpendingInput>),
+    mempool: (Vec<FundingOutput>, Vec<SpendingInput>),
+}
+
+fn calc_balance((funding, spending): &(Vec<FundingOutput>, Vec<SpendingInput>)) -> i64 {
+    let funded: u64 = funding.iter().map(|output| output.value).sum();
+    let spent: u64 = spending.iter().map(|input| input.value).sum();
+    funded as i64 - spent as i64
+}
+
+impl Status {
+    fn funding(&self) -> impl Iterator<Item = &FundingOutput> {
+        self.confirmed.0.iter().chain(self.mempool.0.iter())
+    }
+
+    fn spending(&self) -> impl Iterator<Item = &SpendingInput> {
+        self.confirmed.1.iter().chain(self.mempool.1.iter())
+    }
+
+    pub fn confirmed_balance(&self) -> i64 {
+        calc_balance(&self.confirmed)
+    }
+
+    pub fn mempool_balance(&self) -> i64 {
+        calc_balance(&self.mempool)
+    }
+
+    pub fn history(&self) -> Vec<(u32, Sha256dHash)> {
+        let mut txns_map = HashMap::<Sha256dHash, u32>::new();
+        for f in self.funding() {
+            txns_map.insert(f.txn_id, f.height);
+        }
+        for s in self.spending() {
+            txns_map.insert(s.txn_id, s.height);
+        }
+        let mut txns: Vec<(u32, Sha256dHash)> =
+            txns_map.into_iter().map(|item| (item.1, item.0)).collect();
+        txns.sort_unstable();
+        txns
+    }
+
+    pub fn history_txs(&self) -> Vec<&TxnHeight> {
+        let mut txns_map = BTreeMap::<Sha256dHash, &TxnHeight>::new();
+        for f in self.funding() {
+            txns_map.insert(f.txn_id, &f.txn.as_ref().unwrap());
+        }
+        for s in self.spending() {
+            txns_map.insert(s.txn_id, &s.txn.as_ref().unwrap());
+        }
+        let mut txns: Vec<&TxnHeight> = txns_map.into_iter().map(|item| item.1).collect();
+        // Sort in reverse confirmation height order (unconfirmed txns use u32::max_value as their height):
+        txns.sort_by(|a, b| b.height.cmp(&a.height));
+        txns
+    }
+
+    pub fn unspent(&self) -> Vec<&FundingOutput> {
+        let mut outputs_map = HashMap::<OutPoint, &FundingOutput>::new();
+        for f in self.funding() {
+            outputs_map.insert((f.txn_id, f.output_index), f);
+        }
+        for s in self.spending() {
+            if let None = outputs_map.remove(&s.funding_output) {
+                warn!("failed to remove {:?}", s.funding_output);
+            }
+        }
+        let mut outputs = outputs_map
+            .into_iter()
+            .map(|item| item.1) // a reference to unspent output
+            .collect::<Vec<&FundingOutput>>();
+        outputs.sort_unstable_by_key(|out| out.height);
+        outputs
+    }
+}
+
+#[derive(Clone)]
+pub struct TxnHeight {
+    pub txn: Transaction,
+    pub height: u32,
+    pub blockhash: Sha256dHash,
+}
+
+fn merklize(left: Sha256dHash, right: Sha256dHash) -> Sha256dHash {
+    let data = [&left[..], &right[..]].concat();
+    Sha256dHash::from_data(&data)
+}
+
+fn txrow_by_txid(store: &ReadStore, txid: &Sha256dHash) -> Option<TxRow> {
+    let key = TxRow::filter_full(&txid);
+    let value = store.get(&key)?;
+    Some(TxRow::from_row(&Row { key, value }))
+}
+
+fn rawtxrow_by_txid(store: &ReadStore, txid: &Sha256dHash) -> Option<RawTxRow> {
+    let key = RawTxRow::filter_full(&txid);
+    let value = store.get(&key)?;
+    Some(RawTxRow::from_row(&Row { key, value }))
+}
+
+fn txrows_by_prefix(store: &ReadStore, txid_prefix: &HashPrefix) -> Vec<TxRow> {
+    store
+        .scan(&TxRow::filter_prefix(&txid_prefix))
+        .iter()
+        .map(|row| TxRow::from_row(row))
+        .collect()
+}
+
+fn txids_by_script_hash(store: &ReadStore, script_hash: &[u8]) -> Vec<HashPrefix> {
+    store
+        .scan(&TxOutRow::filter(script_hash))
+        .iter()
+        .take(FUNDING_TXN_LIMIT + 1)
+        .map(|row| TxOutRow::from_row(row).txid_prefix)
+        .collect()
+}
+
+fn txids_by_funding_output(
+    store: &ReadStore,
+    txn_id: &Sha256dHash,
+    output_index: usize,
+) -> Vec<HashPrefix> {
+    store
+        .scan(&TxInRow::filter(&txn_id, output_index))
+        .iter()
+        .map(|row| TxInRow::from_row(row).txid_prefix)
+        .collect()
+}
+
+pub fn get_block_meta(store: &ReadStore, blockhash: &Sha256dHash) -> Option<BlockMeta> {
+    let key = [b"M", &blockhash[..]].concat();
+    let value = store.get(&key)?;
+    let meta: BlockMeta = bincode::deserialize(&value).unwrap();
+    Some(meta)
+}
+
+pub fn get_block_txids(store: &ReadStore, blockhash: &Sha256dHash) -> Option<Vec<Sha256dHash>> {
+    let key = [b"X", &blockhash[..]].concat();
+    let value = store.get(&key)?;
+    let txids: Vec<Sha256dHash> = bincode::deserialize(&value).unwrap();
+    Some(txids)
+}
+
+pub struct Query {
+    app: Arc<App>,
+    tracker: RwLock<Tracker>,
+    extended_db_enabled: bool,
+
+    // monitoring
+    latency: HistogramVec,
+}
+
+impl Query {
+    pub fn new(app: Arc<App>, extended_db_enabled: bool, metrics: &Metrics) -> Arc<Query> {
+        let latency_buckets = vec![
+            1e-4, 2e-4, 5e-4, 1e-3, 2e-3, 5e-3, 1e-2, 2e-2, 5e-2, 0.1, 0.2, 0.5, 1., 2., 5., 10.,
+            20., 50., 100.,
+        ];
+        Arc::new(Query {
+            app,
+            extended_db_enabled,
+            tracker: RwLock::new(Tracker::new(metrics)),
+            latency: metrics.histogram_vec(
+                HistogramOpts::new("query_latency", "Query latency (in seconds)")
+                    .buckets(latency_buckets),
+                &["type"],
+            ),
+        })
+    }
+
+    fn load_txns_by_prefix(
+        &self,
+        store: &ReadStore,
+        prefixes: Vec<HashPrefix>,
+    ) -> Result<Vec<TxnHeight>> {
+        if prefixes.len() > FUNDING_TXN_LIMIT {
+            bail!("Too many txs");
+        }
+
+        let mut txns = vec![];
+        let _timer = self
+            .latency
+            .with_label_values(&["load_txns_by_prefix"])
+            .start_timer();
+        for txid_prefix in prefixes {
+            for tx_row in txrows_by_prefix(store, &txid_prefix) {
+                let txid: Sha256dHash = deserialize(&tx_row.key.txid).unwrap();
+                let txn = self.load_txn(&txid).chain_err(|| "cannot locate tx")?;
+                txns.push(TxnHeight {
+                    txn,
+                    height: tx_row.height,
+                    blockhash: tx_row.blockhash,
+                })
+            }
+        }
+        Ok(txns)
+    }
+
+    fn find_spending_input(
+        &self,
+        store: &ReadStore,
+        funding: &FundingOutput,
+    ) -> Result<Option<SpendingInput>> {
+        let _timer = self
+            .latency
+            .with_label_values(&["find_spending_input"])
+            .start_timer();
+        let spending_txns: Vec<TxnHeight> = self.load_txns_by_prefix(
+            store,
+            txids_by_funding_output(store, &funding.txn_id, funding.output_index),
+        )?;
+        let mut spending_inputs = vec![];
+        for t in &spending_txns {
+            for (input_index, input) in t.txn.input.iter().enumerate() {
+                if input.previous_output.txid == funding.txn_id
+                    && input.previous_output.vout == funding.output_index as u32
+                {
+                    spending_inputs.push(SpendingInput {
+                        txn: Some(t.clone()),
+                        txn_id: t.txn.txid(),
+                        height: t.height,
+                        input_index: input_index,
+                        funding_output: (funding.txn_id, funding.output_index),
+                        value: funding.value,
+                    })
+                }
+            }
+        }
+        assert!(spending_inputs.len() <= 1);
+        Ok(if spending_inputs.len() == 1 {
+            Some(spending_inputs.remove(0))
+        } else {
+            None
+        })
+    }
+
+    fn find_funding_outputs(&self, t: &TxnHeight, script_hash: &[u8]) -> Vec<FundingOutput> {
+        let _timer = self
+            .latency
+            .with_label_values(&["find_funding_outputs"])
+            .start_timer();
+        let mut result = vec![];
+        let txn_id = t.txn.txid();
+        for (index, output) in t.txn.output.iter().enumerate() {
+            if compute_script_hash(&output.script_pubkey[..]) == script_hash {
+                let value = match output.value {
+                    confidential::Value::Explicit(val) => val,
+                    _ => 0,
+                };
+                let asset = match output.asset {
+                    confidential::Asset::Explicit(val) => Some(val),
+                    _ => None,
+                };
+
+                result.push(FundingOutput {
+                    txn: Some(t.clone()),
+                    txn_id: txn_id,
+                    height: t.height,
+                    output_index: index,
+                    value: value,
+                    asset: asset,
+                })
+            }
+        }
+        result
+    }
+
+    fn confirmed_status(
+        &self,
+        script_hash: &[u8],
+    ) -> Result<(Vec<FundingOutput>, Vec<SpendingInput>)> {
+        let _timer = self
+            .latency
+            .with_label_values(&["confirmed_status"])
+            .start_timer();
+        let mut funding = vec![];
+        let mut spending = vec![];
+        let read_store = self.app.read_store();
+        let txid_prefixes = txids_by_script_hash(read_store, script_hash);
+        for t in self.load_txns_by_prefix(read_store, txid_prefixes)? {
+            funding.extend(self.find_funding_outputs(&t, script_hash));
+        }
+        for funding_output in &funding {
+            if let Some(spent) = self.find_spending_input(read_store, &funding_output)? {
+                spending.push(spent);
+            }
+        }
+        Ok((funding, spending))
+    }
+
+    fn mempool_status(
+        &self,
+        script_hash: &[u8],
+        confirmed_funding: &[FundingOutput],
+    ) -> Result<(Vec<FundingOutput>, Vec<SpendingInput>)> {
+        let _timer = self
+            .latency
+            .with_label_values(&["mempool_status"])
+            .start_timer();
+        let mut funding = vec![];
+        let mut spending = vec![];
+        let tracker = self.tracker.read().unwrap();
+        let txid_prefixes = txids_by_script_hash(tracker.index(), script_hash);
+        for t in self.load_txns_by_prefix(tracker.index(), txid_prefixes)? {
+            funding.extend(self.find_funding_outputs(&t, script_hash));
+        }
+        // // TODO: dedup outputs (somehow) both confirmed and in mempool (e.g. reorg?)
+        for funding_output in funding.iter().chain(confirmed_funding.iter()) {
+            if let Some(spent) = self.find_spending_input(tracker.index(), &funding_output)? {
+                spending.push(spent);
+            }
+        }
+        Ok((funding, spending))
+    }
+
+    pub fn status(&self, script_hash: &[u8]) -> Result<Status> {
+        let _timer = self.latency.with_label_values(&["status"]).start_timer();
+        let confirmed = self.confirmed_status(script_hash)?;
+        //.chain_err(|| "failed to get confirmed status")?;
+        let mempool = self.mempool_status(script_hash, &confirmed.0)?;
+        //.chain_err(|| "failed to get mempool status")?;
+        Ok(Status { confirmed, mempool })
+    }
+
+    pub fn find_spending_by_outpoint(&self, outpoint: OutPoint) -> Result<Option<SpendingInput>> {
+        let _timer = self
+            .latency
+            .with_label_values(&["find_spending_by_outpoint"])
+            .start_timer();
+        let funding_output = FundingOutput::from(outpoint);
+        let read_store = self.app.read_store();
+        let tracker = self.tracker.read().unwrap();
+        Ok(
+            if let Some(spent) = self.find_spending_input(read_store, &funding_output)? {
+                Some(spent)
+            } else if let Some(spent) =
+                self.find_spending_input(tracker.index(), &funding_output)?
+            {
+                Some(spent)
+            } else {
+                None
+            },
+        )
+    }
+
+    pub fn find_spending_for_funding_tx(
+        &self,
+        tx: Transaction,
+    ) -> Result<Vec<Option<SpendingInput>>> {
+        let _timer = self
+            .latency
+            .with_label_values(&["find_spending_for_funding_tx"])
+            .start_timer();
+        let txid = tx.txid();
+        let mut spends = vec![];
+        for (output_index, output) in tx.output.iter().enumerate() {
+            let spend = if !output.is_fee() && !output.script_pubkey.is_provably_unspendable() {
+                self.find_spending_by_outpoint((txid, output_index))?
+            } else {
+                None
+            };
+            spends.push(spend)
+        }
+        Ok(spends)
+    }
+
+    // Load transaction by txid
+    pub fn load_txn(&self, txid: &Sha256dHash) -> Result<Transaction> {
+        let _timer = self.latency.with_label_values(&["load_txn"]).start_timer();
+        if self.extended_db_enabled {
+            // fetch from our txstore or mempool tracker
+            rawtxrow_by_txid(self.app.read_store(), txid)
+                .map(|row| deserialize(&row.rawtx).expect("cannot parse tx from txstore"))
+                .or_else(|| self.tracker.read().unwrap().get_txn(&txid))
+                .chain_err(|| format!("cannot find tx {}", txid))
+        } else {
+            // fetch from bitcoind
+            self.app.daemon().gettransaction(txid)
+        }
+    }
+
+    // Load raw transaction by txid
+    pub fn load_raw_txn(&self, txid: &Sha256dHash) -> Result<Bytes> {
+        let _timer = self
+            .latency
+            .with_label_values(&["load_raw_txn"])
+            .start_timer();
+        if self.extended_db_enabled {
+            // fetch from our txstore or mempool tracker
+            Ok(rawtxrow_by_txid(self.app.read_store(), txid)
+                .map(|row| row.rawtx)
+                .or_else(|| {
+                    self.tracker
+                        .read()
+                        .unwrap()
+                        .get_txn(&txid)
+                        .map(|tx| serialize(&tx))
+                })
+                .chain_err(|| format!("cannot find tx {}", txid))?)
+        } else {
+            // fetch from bitcoind
+            let tx_val = self.app.daemon().gettransaction_raw(txid, false)?;
+            Ok(
+                ::hex::decode(tx_val.as_str().chain_err(|| "non-string tx hex")?)
+                    .chain_err(|| "invalid hex")?,
+            )
+        }
+    }
+
+    // Public API for transaction retrieval (for Electrum RPC)
+    // Fetched from bitcoind, includes tx confirmation information (number of confirmations and block hash)
+    pub fn get_transaction(&self, tx_hash: &Sha256dHash, verbose: bool) -> Result<Value> {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_transaction"])
+            .start_timer();
+        self.app.daemon().gettransaction_raw(tx_hash, verbose)
+    }
+
+    pub fn get_block(&self, blockhash: &Sha256dHash) -> Result<Block> {
+        let _timer = self.latency.with_label_values(&["get_block"]).start_timer();
+        self.app.daemon().getblock(blockhash)
+    }
+
+    pub fn get_block_header_with_meta(&self, blockhash: &Sha256dHash) -> Result<BlockHeaderMeta> {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_block_header_with_meta"])
+            .start_timer();
+        Ok(BlockHeaderMeta {
+            header_entry: self.get_header_by_hash(blockhash)?,
+            meta: self.get_block_meta(blockhash)?,
+        })
+    }
+
+    pub fn get_block_txids(&self, blockhash: &Sha256dHash) -> Result<Vec<Sha256dHash>> {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_block_txids"])
+            .start_timer();
+        if self.extended_db_enabled {
+            // fetch from our blockhash=>txids index
+            get_block_txids(self.app.read_store(), blockhash)
+                .chain_err(|| "cannot load block txids")
+        } else {
+            // fetch from bitcoind
+            let block = self
+                .app
+                .daemon()
+                .getblock_raw(blockhash, 1)
+                .chain_err(|| "cannot load block")?;
+            let txids = block
+                .get("tx")
+                .chain_err(|| "block missing txids")?
+                .as_array()
+                .chain_err(|| "invalid block txids")?;
+            Ok(txids
+                .iter()
+                .map(|txid| {
+                    Sha256dHash::from_hex(txid.as_str().chain_err(|| "txid not string")?)
+                        .chain_err(|| "invalid hex")
+                })
+                .collect::<Result<Vec<Sha256dHash>>>()?)
+        }
+    }
+
+    pub fn get_block_meta(&self, blockhash: &Sha256dHash) -> Result<BlockMeta> {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_block_meta"])
+            .start_timer();
+        if self.extended_db_enabled {
+            // fetch from our blockhash=>txids index
+            get_block_meta(self.app.read_store(), blockhash).chain_err(|| "cannot load block meta")
+        } else {
+            // fetch from bitcoind
+            BlockMeta::parse_getblock(self.app.daemon().getblock_raw(blockhash, 1)?)
+        }
+    }
+
+    pub fn get_headers(&self, heights: &[usize]) -> Vec<HeaderEntry> {
+        let index = self.app.index();
+        heights
+            .iter()
+            .filter_map(|height| index.get_header(*height))
+            .collect()
+    }
+
+    pub fn get_header_by_hash(&self, hash: &Sha256dHash) -> Result<HeaderEntry> {
+        let header = self.app.index().get_header_by_hash(hash);
+        Ok(header.chain_err(|| "no header found")?.clone())
+    }
+
+    pub fn get_best_header(&self) -> Result<HeaderEntry> {
+        let last_header = self.app.index().best_header();
+        Ok(last_header.chain_err(|| "no headers indexed")?.clone())
+    }
+
+    pub fn get_best_header_hash(&self) -> Sha256dHash {
+        self.app.index().best_header_hash()
+    }
+
+    pub fn get_best_height(&self) -> usize {
+        self.app.index().best_height()
+    }
+
+    pub fn get_block_status(&self, hash: &Sha256dHash) -> BlockStatus {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_block_status"])
+            .start_timer();
+        // get_header_by_hash looks up the height first, then fetches the header by that.
+        // if the block is no longer the best block at this height, it'll return None.
+        match self.app.index().get_header_by_hash(hash) {
+            Some(header) => BlockStatus {
+                in_best_chain: true,
+                height: Some(header.height()),
+                next_best: self
+                    .app
+                    .index()
+                    .get_header(header.height() + 1)
+                    .map(|h| h.hash().clone()),
+            },
+            None => BlockStatus {
+                in_best_chain: false,
+                height: None,
+                next_best: None,
+            },
+        }
+    }
+
+    pub fn get_tx_status(&self, tx_hash: &Sha256dHash) -> Result<TransactionStatus> {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_tx_status"])
+            .start_timer();
+        // try fetching the height/hash of the block seen to confirm the tx
+        let (height, blockhash) = match txrow_by_txid(self.app.read_store(), &tx_hash) {
+            None => return Ok(TransactionStatus::unconfirmed()),
+            Some(txrow) => (txrow.height, txrow.blockhash),
+        };
+
+        // fetch the block header at the recorded confirmation height
+        let header = self
+            .app
+            .index()
+            .get_header(height as usize)
+            .chain_err(|| "invalid block height for tx")?;
+
+        // the block at confirmation height is not the one containing the tx, must've reorged!
+        if header.hash() != &blockhash {
+            Ok(TransactionStatus::unconfirmed())
+        } else {
+            Ok(TransactionStatus::confirmed(&header))
+        }
+    }
+
+    pub fn get_merkle_proof(
+        &self,
+        tx_hash: &Sha256dHash,
+        block_hash: &Sha256dHash,
+    ) -> Result<(Vec<Sha256dHash>, usize)> {
+        let _timer = self
+            .latency
+            .with_label_values(&["get_merkle_proof"])
+            .start_timer();
+        let mut txids = self
+            .get_block_txids(&block_hash)
+            .chain_err(|| format!("missing txids for block #{}", block_hash))?;
+        let pos = txids
+            .iter()
+            .position(|txid| txid == tx_hash)
+            .chain_err(|| format!("missing txid {}", tx_hash))?;
+        let mut merkle = vec![];
+        let mut index = pos;
+        while txids.len() > 1 {
+            if txids.len() % 2 != 0 {
+                let last = txids.last().unwrap().clone();
+                txids.push(last);
+            }
+            index = if index % 2 == 0 { index + 1 } else { index - 1 };
+            merkle.push(txids[index]);
+            index = index / 2;
+            txids = txids
+                .chunks(2)
+                .map(|pair| merklize(pair[0], pair[1]))
+                .collect()
+        }
+        Ok((merkle, pos))
+    }
+
+    pub fn broadcast(&self, txn: &Transaction) -> Result<Sha256dHash> {
+        self.app.daemon().broadcast(txn)
+    }
+
+    pub fn update_mempool(&self) -> Result<()> {
+        self.tracker.write().unwrap().update(self.app.daemon())
+    }
+
+    /// Returns [vsize, fee_rate] pairs (measured in vbytes and satoshis).
+    pub fn get_fee_histogram(&self) -> Vec<(f32, u32)> {
+        self.tracker.read().unwrap().fee_histogram().clone()
+    }
+
+    // Fee rate [BTC/kB] to be confirmed in `blocks` from now.
+    pub fn estimate_fee(&self, blocks: usize) -> f32 {
+        let mut total_vsize = 0u32;
+        let mut last_fee_rate = 0.0;
+        let blocks_in_vbytes = (blocks * 1_000_000) as u32; // assume ~1MB blocks
+        for (fee_rate, vsize) in self.tracker.read().unwrap().fee_histogram() {
+            last_fee_rate = *fee_rate;
+            total_vsize += vsize;
+            if total_vsize >= blocks_in_vbytes {
+                break; // under-estimate the fee rate a bit
+            }
+        }
+        last_fee_rate * 1e-5 // [BTC/kB] = 10^5 [sat/B]
+    }
+}

src/signal.rs

@@ -1,58 +1,34 @@
-use crossbeam_channel as channel;
-use crossbeam_channel::RecvTimeoutError;
-use std::time::{Duration, Instant};
+use chan;
+use chan_signal;
+use std::time::Duration;
 
-use signal_hook::consts::{SIGINT, SIGTERM, SIGUSR1};
-
-use crate::errors::*;
+use errors::*;
 
 #[derive(Clone)] // so multiple threads could wait on signals
 pub struct Waiter {
-    receiver: channel::Receiver<i32>,
-}
-
-fn notify(signals: &[i32]) -> channel::Receiver<i32> {
-    let (s, r) = channel::bounded(1);
-    let mut signals =
-        signal_hook::iterator::Signals::new(signals).expect("failed to register signal hook");
-    crate::util::spawn_thread("signal-notifier", move || {
-        for signal in signals.forever() {
-            s.send(signal)
-                .unwrap_or_else(|_| panic!("failed to send signal {}", signal));
-        }
-    });
-    r
+    signal: chan::Receiver<chan_signal::Signal>,
 }
 
 impl Waiter {
-    pub fn start() -> Waiter {
+    pub fn new() -> Waiter {
         Waiter {
-            receiver: notify(&[
-                SIGINT, SIGTERM,
-                SIGUSR1, // allow external triggering (e.g. via bitcoind `blocknotify`)
-            ]),
+            signal: chan_signal::notify(&[chan_signal::Signal::INT, chan_signal::Signal::TERM]),
         }
     }
-
-    pub fn wait(&self, duration: Duration, accept_sigusr: bool) -> Result<()> {
-        // Determine the deadline time based on the duration, so that it doesn't
-        // get pushed back when wait_deadline() recurses
-        self.wait_deadline(Instant::now() + duration, accept_sigusr)
-    }
-
-    fn wait_deadline(&self, deadline: Instant, accept_sigusr: bool) -> Result<()> {
-        match self.receiver.recv_deadline(deadline) {
-            Ok(sig) if sig == SIGUSR1 => {
-                trace!("notified via SIGUSR1");
-                if accept_sigusr {
-                    Ok(())
-                } else {
-                    self.wait_deadline(deadline, accept_sigusr)
+    pub fn wait(&self, duration: Duration) -> Result<()> {
+        let signal = &self.signal;
+        let timeout = chan::after(duration);
+        chan_select! {
+            signal.recv() -> s => {
+                if let Some(sig) = s {
+                    bail!(ErrorKind::Interrupt(sig));
                 }
-            }
-            Ok(sig) => bail!(ErrorKind::Interrupt(sig)),
-            Err(RecvTimeoutError::Timeout) => Ok(()),
-            Err(RecvTimeoutError::Disconnected) => bail!("signal hook channel disconnected"),
+            },
+            timeout.recv() => {},
         }
+        Ok(())
+    }
+    pub fn poll(&self) -> Result<()> {
+        self.wait(Duration::from_secs(0))
     }
 }

src/store.rs

@@ -0,0 +1,221 @@
+use bincode;
+use rocksdb;
+
+use std::path::{Path, PathBuf};
+
+use config::Config;
+use util::Bytes;
+
+const DB_VERSION: u32 = 1;
+
+#[derive(Clone)]
+pub struct Row {
+    pub key: Bytes,
+    pub value: Bytes,
+}
+
+impl Row {
+    pub fn into_pair(self) -> (Bytes, Bytes) {
+        (self.key, self.value)
+    }
+}
+
+pub trait ReadStore: Sync {
+    fn get(&self, key: &[u8]) -> Option<Bytes>;
+    fn scan(&self, prefix: &[u8]) -> Vec<Row>;
+}
+
+pub trait WriteStore: Sync {
+    fn write(&self, rows: Vec<Row>);
+    fn flush(&self);
+}
+
+#[derive(Clone)]
+struct Options {
+    path: PathBuf,
+    bulk_import: bool,
+    low_memory: bool,
+}
+
+pub struct DBStore {
+    db: rocksdb::DB,
+    opts: Options,
+}
+
+impl DBStore {
+    fn open_opts(opts: Options) -> Self {
+        debug!("opening DB at {:?}", opts.path);
+        let mut db_opts = rocksdb::Options::default();
+        db_opts.create_if_missing(true);
+        // db_opts.set_keep_log_file_num(10);
+        db_opts.set_max_open_files(if opts.bulk_import { 16 } else { 256 });
+        db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
+        db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy);
+        db_opts.set_target_file_size_base(256 << 20);
+        db_opts.set_write_buffer_size(256 << 20);
+        db_opts.set_disable_auto_compactions(opts.bulk_import); // for initial bulk load
+        db_opts.set_advise_random_on_open(!opts.bulk_import); // bulk load uses sequential I/O
+        if opts.low_memory == false {
+            db_opts.set_compaction_readahead_size(1 << 20);
+        }
+
+        let mut block_opts = rocksdb::BlockBasedOptions::default();
+        block_opts.set_block_size(if opts.low_memory { 256 << 10 } else { 1 << 20 });
+        DBStore {
+            db: rocksdb::DB::open(&db_opts, &opts.path).unwrap(),
+            opts,
+        }
+    }
+
+    /// Opens a new RocksDB at the specified location.
+    pub fn open(path: &Path, low_memory: bool) -> Self {
+        DBStore::open_opts(Options {
+            path: path.to_path_buf(),
+            bulk_import: true,
+            low_memory,
+        })
+    }
+
+    pub fn enable_compaction(self) -> Self {
+        let mut opts = self.opts.clone();
+        if opts.bulk_import == true {
+            opts.bulk_import = false;
+            drop(self); // DB must be closed before being re-opened
+            info!("enabling auto-compactions");
+            DBStore::open_opts(opts)
+        } else {
+            self
+        }
+    }
+
+    pub fn compact(self) -> Self {
+        let opts = self.opts.clone();
+        drop(self); // DB must be closed before being re-opened
+
+        let store = DBStore::open_opts(opts);
+        info!("starting full compaction");
+        store.db.compact_range(None, None); // would take a while
+        info!("finished full compaction");
+        store
+    }
+
+    pub fn iter_scan(&self, prefix: &[u8]) -> ScanIterator {
+        ScanIterator {
+            prefix: prefix.to_vec(),
+            iter: self.db.prefix_iterator(prefix),
+            done: false,
+        }
+    }
+}
+
+pub struct ScanIterator {
+    prefix: Vec<u8>,
+    iter: rocksdb::DBIterator,
+    done: bool,
+}
+
+impl Iterator for ScanIterator {
+    type Item = Row;
+
+    fn next(&mut self) -> Option<Row> {
+        if self.done {
+            return None;
+        }
+        let (key, value) = self.iter.next()?;
+        if !key.starts_with(&self.prefix) {
+            self.done = true;
+            return None;
+        }
+        Some(Row {
+            key: key.to_vec(),
+            value: value.to_vec(),
+        })
+    }
+}
+
+impl ReadStore for DBStore {
+    fn get(&self, key: &[u8]) -> Option<Bytes> {
+        self.db.get(key).unwrap().map(|v| v.to_vec())
+    }
+
+    // TODO: use generators
+    fn scan(&self, prefix: &[u8]) -> Vec<Row> {
+        let mut rows = vec![];
+        for (key, value) in self.db.iterator(rocksdb::IteratorMode::From(
+            prefix,
+            rocksdb::Direction::Forward,
+        )) {
+            if !key.starts_with(prefix) {
+                break;
+            }
+            rows.push(Row {
+                key: key.to_vec(),
+                value: value.to_vec(),
+            });
+        }
+        rows
+    }
+}
+
+impl WriteStore for DBStore {
+    fn write(&self, rows: Vec<Row>) {
+        let mut batch = rocksdb::WriteBatch::default();
+        for row in rows {
+            batch.put(row.key.as_slice(), row.value.as_slice()).unwrap();
+        }
+        let mut opts = rocksdb::WriteOptions::new();
+        opts.set_sync(!self.opts.bulk_import);
+        opts.disable_wal(self.opts.bulk_import);
+        self.db.write_opt(batch, &opts).unwrap();
+    }
+
+    fn flush(&self) {
+        let mut opts = rocksdb::WriteOptions::new();
+        opts.set_sync(true);
+        opts.disable_wal(false);
+        let empty = rocksdb::WriteBatch::default();
+        self.db.write_opt(empty, &opts).unwrap();
+    }
+}
+
+impl Drop for DBStore {
+    fn drop(&mut self) {
+        trace!("closing DB at {:?}", self.opts.path);
+    }
+}
+
+fn full_compaction_marker() -> Row {
+    Row {
+        key: b"F".to_vec(),
+        value: b"".to_vec(),
+    }
+}
+
+pub fn full_compaction(store: DBStore) -> DBStore {
+    store.flush();
+    let store = store.compact().enable_compaction();
+    store.write(vec![full_compaction_marker()]);
+    store
+}
+
+pub fn is_fully_compacted(store: &ReadStore) -> bool {
+    let marker = store.get(&full_compaction_marker().key);
+    marker.is_some()
+}
+
+pub fn verify_index_compatibility(store: &DBStore, config: &Config) {
+    let compatibility_bytes =
+        bincode::serialize(&(config.network_type, DB_VERSION, config.extended_db_enabled)).unwrap();
+
+    match store.get(b"C") {
+        None => store.write(vec![Row {
+            key: b"C".to_vec(),
+            value: compatibility_bytes,
+        }]),
+        Some(x) => {
+            if x != compatibility_bytes {
+                panic!("Incompatible database found. Changing --light mode requires a reindex.");
+            }
+        }
+    }
+}

src/util.rs

@@ -0,0 +1,438 @@
+use bitcoin::consensus::encode::serialize;
+use bitcoin::util::hash::{BitcoinHash, Sha256dHash};
+use elements::{Block, BlockHeader};
+use errors::*;
+use std::collections::HashMap;
+use std::fmt;
+use std::iter::FromIterator;
+use std::slice;
+use std::sync::mpsc::{channel, sync_channel, Receiver, Sender, SyncSender};
+use std::thread;
+use time;
+
+pub type Bytes = Vec<u8>;
+pub type HeaderMap = HashMap<Sha256dHash, BlockHeader>;
+
+// TODO: consolidate serialization/deserialize code for bincode/bitcoin.
+const HASH_LEN: usize = 32;
+pub const HASH_PREFIX_LEN: usize = 8;
+
+pub type FullHash = [u8; HASH_LEN];
+pub type HashPrefix = [u8; HASH_PREFIX_LEN];
+
+pub fn hash_prefix(hash: &[u8]) -> HashPrefix {
+    array_ref![hash, 0, HASH_PREFIX_LEN].clone()
+}
+
+pub fn full_hash(hash: &[u8]) -> FullHash {
+    array_ref![hash, 0, HASH_LEN].clone()
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TransactionStatus {
+    pub confirmed: bool,
+    pub block_height: Option<usize>,
+    pub block_hash: Option<Sha256dHash>,
+}
+
+impl TransactionStatus {
+    pub fn unconfirmed() -> Self {
+        TransactionStatus {
+            confirmed: false,
+            block_height: None,
+            block_hash: None,
+        }
+    }
+    pub fn confirmed(header: &HeaderEntry) -> Self {
+        TransactionStatus {
+            confirmed: true,
+            block_height: Some(header.height()),
+            block_hash: Some(header.hash().clone()),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct BlockStatus {
+    pub in_best_chain: bool,
+    pub height: Option<usize>,
+    pub next_best: Option<Sha256dHash>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct BlockMeta {
+    pub tx_count: u32,
+    pub size: u32,
+    pub weight: u32,
+}
+
+pub struct BlockHeaderMeta {
+    pub header_entry: HeaderEntry,
+    pub meta: BlockMeta,
+}
+
+impl<'a> From<&'a Block> for BlockMeta {
+    fn from(block: &'a Block) -> BlockMeta {
+        BlockMeta {
+            tx_count: block.txdata.len() as u32,
+            size: serialize(block).len() as u32,
+            weight: block.txdata.iter().map(|tx| tx.get_weight() as u32).sum(),
+        }
+    }
+}
+
+impl BlockMeta {
+    pub fn parse_getblock(val: ::serde_json::Value) -> Result<BlockMeta> {
+        Ok(BlockMeta {
+            tx_count: val
+                .get("nTx")
+                .chain_err(|| "missing nTx")?
+                .as_f64()
+                .chain_err(|| "nTx not a number")? as u32,
+            size: val
+                .get("size")
+                .chain_err(|| "missing size")?
+                .as_f64()
+                .chain_err(|| "size not a number")? as u32,
+            weight: val
+                .get("weight")
+                .chain_err(|| "missing weight")?
+                .as_f64()
+                .chain_err(|| "weight not a number")? as u32,
+        })
+    }
+}
+
+#[derive(Eq, PartialEq, Clone)]
+pub struct HeaderEntry {
+    height: usize,
+    hash: Sha256dHash,
+    header: BlockHeader,
+}
+
+impl HeaderEntry {
+    pub fn hash(&self) -> &Sha256dHash {
+        &self.hash
+    }
+
+    pub fn header(&self) -> &BlockHeader {
+        &self.header
+    }
+
+    pub fn height(&self) -> usize {
+        self.height
+    }
+}
+
+impl fmt::Debug for HeaderEntry {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let last_block_time = time::at_utc(time::Timespec::new(self.header().time as i64, 0))
+            .rfc3339()
+            .to_string();
+        write!(
+            f,
+            "best={} height={} @ {}",
+            self.hash(),
+            self.height(),
+            last_block_time,
+        )
+    }
+}
+
+pub struct HeaderList {
+    headers: Vec<HeaderEntry>,
+    heights: HashMap<Sha256dHash, usize>,
+    tip: Sha256dHash,
+}
+
+impl HeaderList {
+    pub fn empty() -> HeaderList {
+        HeaderList {
+            headers: vec![],
+            heights: HashMap::new(),
+            tip: Sha256dHash::default(),
+        }
+    }
+
+    pub fn order(&self, new_headers: Vec<BlockHeader>) -> Vec<HeaderEntry> {
+        // header[i] -> header[i-1] (i.e. header.last() is the tip)
+        struct HashedHeader {
+            blockhash: Sha256dHash,
+            header: BlockHeader,
+        }
+        let hashed_headers =
+            Vec::<HashedHeader>::from_iter(new_headers.into_iter().map(|header| HashedHeader {
+                blockhash: header.bitcoin_hash(),
+                header,
+            }));
+        for i in 1..hashed_headers.len() {
+            assert_eq!(
+                hashed_headers[i].header.prev_blockhash,
+                hashed_headers[i - 1].blockhash
+            );
+        }
+        let prev_blockhash = match hashed_headers.first() {
+            Some(h) => h.header.prev_blockhash,
+            None => return vec![], // hashed_headers is empty
+        };
+        let null_hash = Sha256dHash::default();
+        let new_height: usize = if prev_blockhash == null_hash {
+            0
+        } else {
+            self.header_by_blockhash(&prev_blockhash)
+                .expect(&format!("{} is not part of the blockchain", prev_blockhash))
+                .height()
+                + 1
+        };
+        (new_height..)
+            .zip(hashed_headers.into_iter())
+            .map(|(height, hashed_header)| HeaderEntry {
+                height: height,
+                hash: hashed_header.blockhash,
+                header: hashed_header.header,
+            })
+            .collect()
+    }
+
+    pub fn apply(&mut self, new_headers: Vec<HeaderEntry>) {
+        // new_headers[i] -> new_headers[i - 1] (i.e. new_headers.last() is the tip)
+        for i in 1..new_headers.len() {
+            assert_eq!(new_headers[i - 1].height() + 1, new_headers[i].height());
+            assert_eq!(
+                *new_headers[i - 1].hash(),
+                new_headers[i].header().prev_blockhash
+            );
+        }
+        let new_height = match new_headers.first() {
+            Some(entry) => {
+                let height = entry.height();
+                let expected_prev_blockhash = if height > 0 {
+                    *self.headers[height - 1].hash()
+                } else {
+                    Sha256dHash::default()
+                };
+                assert_eq!(entry.header().prev_blockhash, expected_prev_blockhash);
+                height
+            }
+            None => return,
+        };
+        debug!(
+            "applying {} new headers from height {}",
+            new_headers.len(),
+            new_height
+        );
+        self.headers.split_off(new_height); // keep [0..new_height) entries
+        for new_header in new_headers {
+            let height = new_header.height();
+            assert_eq!(height, self.headers.len());
+            self.tip = *new_header.hash();
+            self.headers.push(new_header);
+            self.heights.insert(self.tip, height);
+        }
+    }
+
+    pub fn header_by_blockhash(&self, blockhash: &Sha256dHash) -> Option<&HeaderEntry> {
+        let height = self.heights.get(blockhash)?;
+        let header = self.headers.get(*height)?;
+        if *blockhash == *header.hash() {
+            Some(header)
+        } else {
+            None
+        }
+    }
+
+    pub fn header_by_height(&self, height: usize) -> Option<&HeaderEntry> {
+        self.headers.get(height).map(|entry| {
+            assert_eq!(entry.height(), height);
+            entry
+        })
+    }
+
+    pub fn equals(&self, other: &HeaderList) -> bool {
+        self.headers.last() == other.headers.last()
+    }
+
+    pub fn tip(&self) -> &Sha256dHash {
+        assert_eq!(
+            self.tip,
+            self.headers
+                .last()
+                .map(|h| *h.hash())
+                .unwrap_or(Sha256dHash::default())
+        );
+        &self.tip
+    }
+
+    pub fn len(&self) -> usize {
+        self.headers.len()
+    }
+
+    pub fn iter(&self) -> slice::Iter<HeaderEntry> {
+        self.headers.iter()
+    }
+}
+
+pub struct SyncChannel<T> {
+    tx: SyncSender<T>,
+    rx: Receiver<T>,
+}
+
+impl<T> SyncChannel<T> {
+    pub fn new(size: usize) -> SyncChannel<T> {
+        let (tx, rx) = sync_channel(size);
+        SyncChannel { tx, rx }
+    }
+
+    pub fn sender(&self) -> SyncSender<T> {
+        self.tx.clone()
+    }
+
+    pub fn receiver(&self) -> &Receiver<T> {
+        &self.rx
+    }
+
+    pub fn into_receiver(self) -> Receiver<T> {
+        self.rx
+    }
+}
+
+pub struct Channel<T> {
+    tx: Sender<T>,
+    rx: Receiver<T>,
+}
+
+impl<T> Channel<T> {
+    pub fn new() -> Channel<T> {
+        let (tx, rx) = channel();
+        Channel { tx, rx }
+    }
+
+    pub fn sender(&self) -> Sender<T> {
+        self.tx.clone()
+    }
+
+    pub fn receiver(&self) -> &Receiver<T> {
+        &self.rx
+    }
+
+    pub fn into_receiver(self) -> Receiver<T> {
+        self.rx
+    }
+}
+
+pub fn spawn_thread<F, T>(name: &str, f: F) -> thread::JoinHandle<T>
+where
+    F: FnOnce() -> T,
+    F: Send + 'static,
+    T: Send + 'static,
+{
+    thread::Builder::new()
+        .name(name.to_owned())
+        .spawn(f)
+        .unwrap()
+}
+
+use bitcoin::util::hash::Hash160;
+use bitcoin::Script;
+use bitcoin_bech32::constants::Network as B32Network;
+use bitcoin_bech32::{u5, WitnessProgram};
+use daemon::Network;
+use utils::address::{Address, Payload};
+
+// @XXX we can't use any of the Address:p2{...}h utility methods, since they expect the pre-image data, which we don't have.
+// we must instead create the Payload manually, which results in code duplication with the p2{...}h methods, especially for witness programs.
+// ideally, this should be implemented as part of the rust-bitcoin lib.
+pub fn script_to_address(script: &Script, network: &Network) -> Option<String> {
+    let payload = if script.is_p2pkh() {
+        Some(Payload::PubkeyHash(Hash160::from(&script[3..23])))
+    } else if script.is_p2sh() {
+        Some(Payload::ScriptHash(Hash160::from(&script[2..22])))
+    } else if script.is_v0_p2wpkh() {
+        Some(Payload::WitnessProgram(
+            WitnessProgram::new(
+                u5::try_from_u8(0).expect("0<32"),
+                script[2..22].to_vec(),
+                B32Network::from(network),
+            )
+            .unwrap(),
+        ))
+    } else if script.is_v0_p2wsh() {
+        Some(Payload::WitnessProgram(
+            WitnessProgram::new(
+                u5::try_from_u8(0).expect("0<32"),
+                script[2..34].to_vec(),
+                B32Network::from(network),
+            )
+            .unwrap(),
+        ))
+    } else {
+        None
+    };
+
+    Some(
+        Address {
+            payload: payload?,
+            network: *network,
+        }
+        .to_string(),
+    )
+}
+
+use bitcoin::blockdata::script::Instruction::PushBytes;
+use hex;
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct PegOutRequest {
+    pub genesis_hash: String,
+    pub scriptpubkey: Script,
+    pub scriptpubkey_asm: String,
+    pub scriptpubkey_address: Option<String>,
+}
+
+impl PegOutRequest {
+    pub fn parse(
+        script: &Script,
+        parent_network: &Network,
+        parent_genesis_hash: &str,
+    ) -> Option<PegOutRequest> {
+        if !script.is_op_return() {
+            return None;
+        }
+
+        let nulldata: Vec<_> = script.iter(true).skip(1).collect();
+        if nulldata.len() < 2 {
+            return None;
+        }
+
+        let genesis_hash = if let PushBytes(data) = nulldata[0] {
+            hex::encode(data.to_vec())
+        } else {
+            return None;
+        };
+
+        let scriptpubkey = if let PushBytes(data) = nulldata[1] {
+            Script::from(data.to_vec())
+        } else {
+            return None;
+        };
+
+        if genesis_hash != parent_genesis_hash {
+            return None;
+        }
+
+        let scriptpubkey_asm = get_script_asm(&scriptpubkey);
+        let scriptpubkey_address = script_to_address(&scriptpubkey, parent_network);
+
+        Some(PegOutRequest {
+            genesis_hash,
+            scriptpubkey,
+            scriptpubkey_asm,
+            scriptpubkey_address,
+        })
+    }
+}
+
+pub fn get_script_asm(script: &Script) -> String {
+    let asm = format!("{:?}", script);
+    (&asm[7..asm.len() - 1]).to_string()
+}

src/util/bincode_tests.rs

@@ -1,112 +0,0 @@
-/*
-
-The tests below show us the following defaults for each method of using bincode.
-
-1. Using bincode::[de]serialize() directly: "function"
-2. Using bincode::config().[de]serialize(): "Config" (deprecated)
-3. Using bincode::options().[de]serialize(): "Options" (currently recommended for v1.3.3)
-
-```
-+----------+--------+------------+----------------+------------+
-|          | Endian | Int Length | Allow Trailing | Byte Limit |
-+----------+--------+------------+----------------+------------+
-| function | little | fixed      | allow          | unlimited  |
-| Config   | little | fixed      | allow          | unlimited  |
-| Options  | little | variable * | reject *       | unlimited  |
-+----------+--------+------------+----------------+------------+
-```
-
-Thus we only need to change the int length from variable to fixed,
-and allow trailing to allow in order to match the previous behavior.
-(note: TxHistory was using Big Endian by explicitly setting it to big.)
-
-*/
-
-use bincode_do_not_use_directly as bincode;
-
-#[test]
-fn bincode_settings() {
-    let value = TestStruct::new();
-    let mut large = [0_u8; 4096];
-    let decoded = [
-        8_u8, 7, 6, 5, 4, 3, 2, 1, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 8, 7, 6, 5, 4, 3, 2, 1, 1,
-        2, 3, 4, 5, 6, 7, 8, 12, 0, 0, 0, 0, 0, 0, 0, 72, 101, 108, 108, 111, 32, 87, 111, 114,
-        108, 100, 33,
-    ];
-    large[0..56].copy_from_slice(&decoded);
-
-    // Using functions: Little endian, Fixint, Allow trailing, Unlimited
-    assert_eq!(bincode::serialize(&value).unwrap(), &decoded);
-    assert_eq!(bincode::deserialize::<TestStruct>(&large).unwrap(), value);
-
-    // Using Config (deprecated)
-    // Little endian, fixint, Allow trailing, Unlimited
-    #[allow(deprecated)]
-    {
-        assert_eq!(bincode::config().serialize(&value).unwrap(), &decoded);
-        assert_eq!(
-            bincode::config().deserialize::<TestStruct>(&large).unwrap(),
-            value
-        );
-    }
-
-    // Using Options
-    // Little endian, VARINT (different), Reject trailing (different), unlimited
-    use bincode::Options;
-    assert_eq!(
-        bincode::options()
-            .with_fixint_encoding()
-            .allow_trailing_bytes()
-            .serialize(&value)
-            .unwrap(),
-        &decoded
-    );
-    assert_eq!(
-        bincode::options()
-            .with_fixint_encoding()
-            .allow_trailing_bytes()
-            .deserialize::<TestStruct>(&large)
-            .unwrap(),
-        value
-    );
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-struct TestStruct {
-    a: u64,
-    b: [u8; 8],
-    c: TestData,
-    d: String,
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-enum TestData {
-    Foo(FooStruct),
-    Bar(BarStruct),
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-struct FooStruct {
-    a: u64,
-    b: [u8; 8],
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
-struct BarStruct {
-    a: u64,
-    b: [u8; 8],
-}
-
-impl TestStruct {
-    fn new() -> Self {
-        Self {
-            a: 0x0102030405060708,
-            b: [1, 2, 3, 4, 5, 6, 7, 8],
-            c: TestData::Foo(FooStruct {
-                a: 0x0102030405060708,
-                b: [1, 2, 3, 4, 5, 6, 7, 8],
-            }),
-            d: String::from("Hello World!"),
-        }
-    }
-}

src/util/bincode_util.rs

@@ -1,70 +0,0 @@
-//! This module creates two sets of serialize and deserialize for bincode.
-//! They explicitly spell out the bincode settings so that switching to
-//! new versions in the future is less error prone.
-//!
-//! This is a list of all the row types and their settings for bincode.
-//! +--------------+--------+------------+----------------+------------+
-//! |              | Endian | Int Length | Allow Trailing | Byte Limit |
-//! +--------------+--------+------------+----------------+------------+
-//! | TxHistoryRow | big    | fixed      | allow          | unlimited  |
-//! | All others   | little | fixed      | allow          | unlimited  |
-//! +--------------+--------+------------+----------------+------------+
-
-// We only want people to use bincode_util
-use bincode::Options;
-use bincode_do_not_use_directly as bincode;
-
-pub fn serialize_big<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
-where
-    T: ?Sized + serde::Serialize,
-{
-    big_endian().serialize(value)
-}
-
-pub fn deserialize_big<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
-where
-    T: serde::Deserialize<'a>,
-{
-    big_endian().deserialize(bytes)
-}
-
-pub fn serialize_little<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
-where
-    T: ?Sized + serde::Serialize,
-{
-    little_endian().serialize(value)
-}
-
-pub fn deserialize_little<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
-where
-    T: serde::Deserialize<'a>,
-{
-    little_endian().deserialize(bytes)
-}
-
-/// This is the default settings for Options,
-/// but all explicitly spelled out, except for endianness.
-/// The following functions will add endianness.
-#[inline]
-fn options() -> impl Options {
-    bincode::options()
-        .with_fixint_encoding()
-        .with_no_limit()
-        .allow_trailing_bytes()
-}
-
-/// Adding the endian flag for big endian
-#[inline]
-fn big_endian() -> impl Options {
-    options().with_big_endian()
-}
-
-/// Adding the endian flag for little endian
-#[inline]
-fn little_endian() -> impl Options {
-    options().with_little_endian()
-}
-
-#[cfg(test)]
-#[path = "./bincode_tests.rs"]
-mod bincode_tests;

src/util/block.rs

@@ -1,346 +0,0 @@
-use bitcoin::hashes::Hash;
-
-use crate::chain::{BlockHash, BlockHeader};
-use crate::errors::*;
-use crate::new_index::BlockEntry;
-
-use std::collections::HashMap;
-use std::fmt;
-use std::iter::FromIterator;
-use std::slice;
-use time::format_description::well_known::Rfc3339;
-use time::OffsetDateTime as DateTime;
-
-const MTP_SPAN: usize = 11;
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct BlockId {
-    pub height: usize,
-    pub hash: BlockHash,
-    pub time: u32,
-}
-
-impl From<&HeaderEntry> for BlockId {
-    fn from(header: &HeaderEntry) -> Self {
-        BlockId {
-            height: header.height(),
-            hash: *header.hash(),
-            time: header.header().time,
-        }
-    }
-}
-
-#[derive(Eq, PartialEq, Clone)]
-pub struct HeaderEntry {
-    height: usize,
-    hash: BlockHash,
-    header: BlockHeader,
-}
-
-impl HeaderEntry {
-    pub fn hash(&self) -> &BlockHash {
-        &self.hash
-    }
-
-    pub fn header(&self) -> &BlockHeader {
-        &self.header
-    }
-
-    pub fn height(&self) -> usize {
-        self.height
-    }
-}
-
-impl fmt::Debug for HeaderEntry {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let last_block_time = DateTime::from_unix_timestamp(self.header().time as i64).unwrap();
-        write!(
-            f,
-            "hash={} height={} @ {}",
-            self.hash(),
-            self.height(),
-            last_block_time.format(&Rfc3339).unwrap(),
-        )
-    }
-}
-
-pub struct HeaderList {
-    headers: Vec<HeaderEntry>,
-    heights: HashMap<BlockHash, usize>,
-    tip: BlockHash,
-}
-
-impl HeaderList {
-    pub fn empty() -> HeaderList {
-        HeaderList {
-            headers: vec![],
-            heights: HashMap::new(),
-            tip: BlockHash::all_zeros(),
-        }
-    }
-
-    pub fn new(
-        mut headers_map: HashMap<BlockHash, BlockHeader>,
-        tip_hash: BlockHash,
-    ) -> HeaderList {
-        trace!(
-            "processing {} headers, tip at {:?}",
-            headers_map.len(),
-            tip_hash
-        );
-
-        let mut blockhash = tip_hash;
-        let mut headers_chain: Vec<BlockHeader> = vec![];
-        let null_hash = BlockHash::all_zeros();
-
-        while blockhash != null_hash {
-            let header = headers_map.remove(&blockhash).unwrap_or_else(|| {
-                panic!(
-                    "missing expected blockhash in headers map: {:?}, pointed from: {:?}",
-                    blockhash,
-                    headers_chain.last().map(|h| h.block_hash())
-                )
-            });
-            blockhash = header.prev_blockhash;
-            headers_chain.push(header);
-        }
-        headers_chain.reverse();
-
-        trace!(
-            "{} chained headers ({} orphan blocks left)",
-            headers_chain.len(),
-            headers_map.len()
-        );
-
-        let mut headers = HeaderList::empty();
-        headers.apply(headers.order(headers_chain));
-        headers
-    }
-
-    pub fn order(&self, new_headers: Vec<BlockHeader>) -> Vec<HeaderEntry> {
-        // header[i] -> header[i-1] (i.e. header.last() is the tip)
-        struct HashedHeader {
-            blockhash: BlockHash,
-            header: BlockHeader,
-        }
-        let hashed_headers =
-            Vec::<HashedHeader>::from_iter(new_headers.into_iter().map(|header| HashedHeader {
-                blockhash: header.block_hash(),
-                header,
-            }));
-        for i in 1..hashed_headers.len() {
-            assert_eq!(
-                hashed_headers[i].header.prev_blockhash,
-                hashed_headers[i - 1].blockhash
-            );
-        }
-        let prev_blockhash = match hashed_headers.first() {
-            Some(h) => h.header.prev_blockhash,
-            None => return vec![], // hashed_headers is empty
-        };
-        let null_hash = BlockHash::all_zeros();
-        let new_height: usize = if prev_blockhash == null_hash {
-            0
-        } else {
-            self.header_by_blockhash(&prev_blockhash)
-                .unwrap_or_else(|| panic!("{} is not part of the blockchain", prev_blockhash))
-                .height()
-                + 1
-        };
-        (new_height..)
-            .zip(hashed_headers)
-            .map(|(height, hashed_header)| HeaderEntry {
-                height,
-                hash: hashed_header.blockhash,
-                header: hashed_header.header,
-            })
-            .collect()
-    }
-
-    /// Returns any rolled back blocks in order from old tip first and first block in the fork is last
-    /// It also returns the blockhash of the post-rollback tip.
-    pub fn apply(
-        &mut self,
-        new_headers: Vec<HeaderEntry>,
-    ) -> (Vec<HeaderEntry>, Option<BlockHash>) {
-        // new_headers[i] -> new_headers[i - 1] (i.e. new_headers.last() is the tip)
-        for i in 1..new_headers.len() {
-            assert_eq!(new_headers[i - 1].height() + 1, new_headers[i].height());
-            assert_eq!(
-                *new_headers[i - 1].hash(),
-                new_headers[i].header().prev_blockhash
-            );
-        }
-        let new_height = match new_headers.first() {
-            Some(entry) => {
-                let height = entry.height();
-                let expected_prev_blockhash = if height > 0 {
-                    *self.headers[height - 1].hash()
-                } else {
-                    BlockHash::all_zeros()
-                };
-                assert_eq!(entry.header().prev_blockhash, expected_prev_blockhash);
-                height
-            }
-            None => return (vec![], None),
-        };
-        debug!(
-            "applying {} new headers from height {}",
-            new_headers.len(),
-            new_height
-        );
-        let mut removed = self.headers.split_off(new_height); // keep [0..new_height) entries
-
-        // If we reorged, we should return the last blockhash before adding the new chain's blockheaders.
-        let reorged_tip = if !removed.is_empty() {
-            self.headers.last().map(|be| be.hash()).cloned()
-        } else {
-            None
-        };
-
-        for new_header in new_headers {
-            let height = new_header.height();
-            assert_eq!(height, self.headers.len());
-            self.tip = *new_header.hash();
-            self.headers.push(new_header);
-            self.heights.insert(self.tip, height);
-        }
-        removed.reverse();
-        (removed, reorged_tip)
-    }
-
-    pub fn header_by_blockhash(&self, blockhash: &BlockHash) -> Option<&HeaderEntry> {
-        let height = self.heights.get(blockhash)?;
-        let header = self.headers.get(*height)?;
-        if *blockhash == *header.hash() {
-            Some(header)
-        } else {
-            None
-        }
-    }
-
-    pub fn header_by_height(&self, height: usize) -> Option<&HeaderEntry> {
-        self.headers.get(height).inspect(|entry| {
-            assert_eq!(entry.height(), height);
-        })
-    }
-
-    pub fn equals(&self, other: &HeaderList) -> bool {
-        self.headers.last() == other.headers.last()
-    }
-
-    pub fn tip(&self) -> &BlockHash {
-        assert_eq!(
-            self.tip,
-            self.headers
-                .last()
-                .map(|h| *h.hash())
-                .unwrap_or(BlockHash::all_zeros())
-        );
-        &self.tip
-    }
-
-    pub fn len(&self) -> usize {
-        self.headers.len()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.headers.is_empty()
-    }
-
-    pub fn iter(&self) -> slice::Iter<'_, HeaderEntry> {
-        self.headers.iter()
-    }
-
-    /// Get the Median Time Past
-    pub fn get_mtp(&self, height: usize) -> u32 {
-        // Use the timestamp as the mtp of the genesis block.
-        // Matches bitcoind's behaviour: bitcoin-cli getblock `bitcoin-cli getblockhash 0` | jq '.time == .mediantime'
-        if height == 0 {
-            self.headers.first().unwrap().header.time
-        } else if height > self.len() - 1 {
-            0
-        } else {
-            let mut timestamps = (height.saturating_sub(MTP_SPAN - 1)..=height)
-                .map(|p_height| self.headers.get(p_height).unwrap().header.time)
-                .collect::<Vec<_>>();
-            timestamps.sort_unstable();
-            timestamps[timestamps.len() / 2]
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct BlockStatus {
-    pub in_best_chain: bool,
-    pub height: Option<usize>,
-    pub next_best: Option<BlockHash>,
-}
-
-impl BlockStatus {
-    pub fn confirmed(height: usize, next_best: Option<BlockHash>) -> BlockStatus {
-        BlockStatus {
-            in_best_chain: true,
-            height: Some(height),
-            next_best,
-        }
-    }
-
-    pub fn orphaned() -> BlockStatus {
-        BlockStatus {
-            in_best_chain: false,
-            height: None,
-            next_best: None,
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct BlockMeta {
-    #[serde(alias = "nTx")]
-    pub tx_count: u32,
-    pub size: u32,
-    pub weight: u32,
-}
-
-pub struct BlockHeaderMeta {
-    pub header_entry: HeaderEntry,
-    pub meta: BlockMeta,
-    pub mtp: u32,
-}
-
-impl From<&BlockEntry> for BlockMeta {
-    fn from(b: &BlockEntry) -> BlockMeta {
-        #[cfg(not(feature = "liquid"))]
-        let weight = b.block.weight().to_wu() as u32;
-        #[cfg(feature = "liquid")]
-        let weight = b.block.weight() as u32;
-        BlockMeta {
-            tx_count: b.block.txdata.len() as u32,
-            weight,
-            size: b.size,
-        }
-    }
-}
-
-impl BlockMeta {
-    pub fn parse_getblock(val: ::serde_json::Value) -> Result<BlockMeta> {
-        Ok(BlockMeta {
-            tx_count: val
-                .get("nTx")
-                .chain_err(|| "missing nTx")?
-                .as_f64()
-                .chain_err(|| "nTx not a number")? as u32,
-            size: val
-                .get("size")
-                .chain_err(|| "missing size")?
-                .as_f64()
-                .chain_err(|| "size not a number")? as u32,
-            weight: val
-                .get("weight")
-                .chain_err(|| "missing weight")?
-                .as_f64()
-                .chain_err(|| "weight not a number")? as u32,
-        })
-    }
-}

src/util/electrum_merkle.rs

@@ -1,105 +0,0 @@
-use crate::chain::{BlockHash, Txid};
-use crate::errors::*;
-use crate::new_index::ChainQuery;
-use bitcoin::hashes::{sha256d::Hash as Sha256dHash, Hash};
-
-pub fn get_tx_merkle_proof(
-    chain: &ChainQuery,
-    tx_hash: &Txid,
-    block_hash: &BlockHash,
-) -> Result<(Vec<Sha256dHash>, usize)> {
-    let txids = chain
-        .get_block_txids(block_hash)
-        .chain_err(|| format!("missing block txids for #{}", block_hash))?;
-    let pos = txids
-        .iter()
-        .position(|txid| txid == tx_hash)
-        .chain_err(|| format!("missing txid {}", tx_hash))?;
-    let txids = txids.into_iter().map(Sha256dHash::from).collect();
-
-    let (branch, _root) = create_merkle_branch_and_root(txids, pos);
-    Ok((branch, pos))
-}
-
-pub fn get_header_merkle_proof(
-    chain: &ChainQuery,
-    height: usize,
-    cp_height: usize,
-) -> Result<(Vec<Sha256dHash>, Sha256dHash)> {
-    if cp_height < height {
-        bail!("cp_height #{} < height #{}", cp_height, height);
-    }
-
-    let best_height = chain.best_height();
-    if best_height < cp_height {
-        bail!(
-            "cp_height #{} above best block height #{}",
-            cp_height,
-            best_height
-        );
-    }
-
-    let heights: Vec<usize> = (0..=cp_height).collect();
-    let header_hashes: Vec<BlockHash> = heights
-        .into_iter()
-        .map(|height| chain.hash_by_height(height))
-        .collect::<Option<Vec<BlockHash>>>()
-        .chain_err(|| "missing block headers")?;
-
-    let header_hashes = header_hashes.into_iter().map(Sha256dHash::from).collect();
-    Ok(create_merkle_branch_and_root(header_hashes, height))
-}
-
-pub fn get_id_from_pos(
-    chain: &ChainQuery,
-    height: usize,
-    tx_pos: usize,
-    want_merkle: bool,
-) -> Result<(Txid, Vec<Sha256dHash>)> {
-    let header_hash = chain
-        .hash_by_height(height)
-        .chain_err(|| format!("missing block #{}", height))?;
-
-    let txids = chain
-        .get_block_txids(&header_hash)
-        .chain_err(|| format!("missing block txids #{}", height))?;
-
-    let txid = *txids
-        .get(tx_pos)
-        .chain_err(|| format!("No tx in position #{} in block #{}", tx_pos, height))?;
-
-    let txids = txids.into_iter().map(Sha256dHash::from).collect();
-
-    let branch = if want_merkle {
-        create_merkle_branch_and_root(txids, tx_pos).0
-    } else {
-        vec![]
-    };
-    Ok((txid, branch))
-}
-
-fn merklize(left: Sha256dHash, right: Sha256dHash) -> Sha256dHash {
-    let data = [&left[..], &right[..]].concat();
-    Sha256dHash::hash(&data)
-}
-
-fn create_merkle_branch_and_root(
-    mut hashes: Vec<Sha256dHash>,
-    mut index: usize,
-) -> (Vec<Sha256dHash>, Sha256dHash) {
-    let mut merkle = vec![];
-    while hashes.len() > 1 {
-        if hashes.len() % 2 != 0 {
-            let last = *hashes.last().unwrap();
-            hashes.push(last);
-        }
-        index = if index % 2 == 0 { index + 1 } else { index - 1 };
-        merkle.push(hashes[index]);
-        index /= 2;
-        hashes = hashes
-            .chunks(2)
-            .map(|pair| merklize(pair[0], pair[1]))
-            .collect()
-    }
-    (merkle, hashes[0])
-}

src/util/fees.rs

@@ -1,66 +0,0 @@
-use crate::chain::{Network, Transaction, TxOut};
-use std::collections::HashMap;
-
-const VSIZE_BIN_WIDTH: u32 = 50_000; // in vbytes
-
-pub struct TxFeeInfo {
-    pub fee: u64,   // in satoshis
-    pub vsize: u32, // in virtual bytes (= weight/4)
-    pub fee_per_vbyte: f32,
-}
-
-impl TxFeeInfo {
-    pub fn new(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>, network: Network) -> Self {
-        let fee = get_tx_fee(tx, prevouts, network);
-        #[cfg(not(feature = "liquid"))]
-        let vsize = tx.weight().to_wu() / 4;
-        #[cfg(feature = "liquid")]
-        let vsize = tx.weight() / 4;
-
-        TxFeeInfo {
-            fee,
-            vsize: vsize as u32,
-            fee_per_vbyte: fee as f32 / vsize as f32,
-        }
-    }
-}
-
-#[cfg(not(feature = "liquid"))]
-pub fn get_tx_fee(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>, _network: Network) -> u64 {
-    if tx.is_coinbase() {
-        return 0;
-    }
-
-    let total_in: u64 = prevouts
-        .values()
-        .map(|prevout| prevout.value.to_sat())
-        .sum();
-    let total_out: u64 = tx.output.iter().map(|vout| vout.value.to_sat()).sum();
-    total_in - total_out
-}
-
-#[cfg(feature = "liquid")]
-pub fn get_tx_fee(tx: &Transaction, _prevouts: &HashMap<u32, &TxOut>, network: Network) -> u64 {
-    tx.fee_in(*network.native_asset())
-}
-
-pub fn make_fee_histogram(mut entries: Vec<&TxFeeInfo>) -> Vec<(f32, u32)> {
-    entries.sort_unstable_by(|e1, e2| e1.fee_per_vbyte.partial_cmp(&e2.fee_per_vbyte).unwrap());
-
-    let mut histogram = vec![];
-    let mut bin_size = 0;
-    let mut last_fee_rate = 0.0;
-    for e in entries.iter().rev() {
-        if bin_size > VSIZE_BIN_WIDTH && last_fee_rate != e.fee_per_vbyte {
-            // vsize of transactions paying >= last_fee_rate
-            histogram.push((last_fee_rate, bin_size));
-            bin_size = 0;
-        }
-        last_fee_rate = e.fee_per_vbyte;
-        bin_size += e.vsize;
-    }
-    if bin_size > 0 {
-        histogram.push((last_fee_rate, bin_size));
-    }
-    histogram
-}

src/util/mod.rs

@@ -1,229 +0,0 @@
-mod block;
-mod script;
-mod transaction;
-
-pub mod bincode_util;
-pub mod electrum_merkle;
-pub mod fees;
-
-pub use self::block::{BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, HeaderEntry, HeaderList};
-pub use self::fees::get_tx_fee;
-pub use self::script::{
-    get_innerscripts, IsProvablyUnspendable, ScriptToAddr, ScriptToAsm, SegwitDetection,
-};
-pub use self::transaction::{
-    extract_tx_prevouts, has_prevout, is_coinbase, is_spendable, serialize_outpoint,
-    sigops::transaction_sigop_count, TransactionStatus, TxInput,
-};
-
-use std::collections::HashMap;
-use std::sync::atomic::AtomicUsize;
-use std::sync::mpsc::{channel, Receiver, Sender};
-use std::sync::Mutex;
-use std::thread::{self, ThreadId};
-
-use crate::chain::BlockHeader;
-use bitcoin::hashes::sha256d::Hash as Sha256dHash;
-use socket2::{Domain, Protocol, Socket, Type};
-use std::net::SocketAddr;
-
-pub type Bytes = Vec<u8>;
-pub type HeaderMap = HashMap<Sha256dHash, BlockHeader>;
-
-// TODO: consolidate serialization/deserialize code for bincode/bitcoin.
-const HASH_LEN: usize = 32;
-
-pub type FullHash = [u8; HASH_LEN];
-
-pub fn full_hash(hash: &[u8]) -> FullHash {
-    *array_ref![hash, 0, HASH_LEN]
-}
-
-pub struct SyncChannel<T> {
-    tx: Option<crossbeam_channel::Sender<T>>,
-    rx: Option<crossbeam_channel::Receiver<T>>,
-}
-
-impl<T> SyncChannel<T> {
-    pub fn new(size: usize) -> SyncChannel<T> {
-        let (tx, rx) = crossbeam_channel::bounded(size);
-        SyncChannel {
-            tx: Some(tx),
-            rx: Some(rx),
-        }
-    }
-
-    pub fn sender(&self) -> crossbeam_channel::Sender<T> {
-        self.tx.as_ref().expect("No Sender").clone()
-    }
-
-    pub fn receiver(&self) -> &crossbeam_channel::Receiver<T> {
-        self.rx.as_ref().expect("No Receiver")
-    }
-
-    pub fn into_receiver(self) -> crossbeam_channel::Receiver<T> {
-        self.rx.expect("No Receiver")
-    }
-
-    /// This drops the sender and receiver, causing all other methods to panic.
-    ///
-    /// Use only when you know that the channel will no longer be used.
-    /// ie. shutdown.
-    pub fn close(&mut self) -> Option<crossbeam_channel::Receiver<T>> {
-        self.tx.take();
-        self.rx.take()
-    }
-}
-
-pub struct Channel<T> {
-    tx: Sender<T>,
-    rx: Receiver<T>,
-}
-
-impl<T> Channel<T> {
-    pub fn unbounded() -> Self {
-        let (tx, rx) = channel();
-        Channel { tx, rx }
-    }
-
-    pub fn sender(&self) -> Sender<T> {
-        self.tx.clone()
-    }
-
-    pub fn receiver(&self) -> &Receiver<T> {
-        &self.rx
-    }
-
-    pub fn into_receiver(self) -> Receiver<T> {
-        self.rx
-    }
-}
-
-/// This static HashMap contains all the threads spawned with [`spawn_thread`] with their name
-#[inline]
-pub fn with_spawned_threads<F>(f: F)
-where
-    F: FnOnce(&mut HashMap<ThreadId, String>),
-{
-    lazy_static! {
-        static ref SPAWNED_THREADS: Mutex<HashMap<ThreadId, String>> = Mutex::new(HashMap::new());
-    }
-    let mut lock = match SPAWNED_THREADS.lock() {
-        Ok(threads) => threads,
-        // There's no possible broken state
-        Err(threads) => {
-            warn!("SPAWNED_THREADS is in a poisoned state! Be wary of incorrect logs!");
-            threads.into_inner()
-        }
-    };
-    f(&mut lock)
-}
-
-pub fn spawn_thread<F, T>(prefix: &str, do_work: F) -> thread::JoinHandle<T>
-where
-    F: FnOnce() -> T,
-    F: Send + 'static,
-    T: Send + 'static,
-{
-    static THREAD_COUNTER: AtomicUsize = AtomicUsize::new(0);
-    let counter = THREAD_COUNTER.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
-    thread::Builder::new()
-        .name(format!("{}-{}", prefix, counter))
-        .spawn(move || {
-            let thread = std::thread::current();
-            let name = thread.name().unwrap();
-            let id = thread.id();
-
-            trace!("[THREAD] GETHASHMAP INSERT | {name} {id:?}");
-            with_spawned_threads(|threads| {
-                threads.insert(id, name.to_owned());
-            });
-            trace!("[THREAD] START WORK        | {name} {id:?}");
-
-            let result = do_work();
-
-            trace!("[THREAD] FINISHED WORK     | {name} {id:?}");
-            trace!("[THREAD] GETHASHMAP REMOVE | {name} {id:?}");
-            with_spawned_threads(|threads| {
-                threads.remove(&id);
-            });
-            trace!("[THREAD] HASHMAP REMOVED   | {name} {id:?}");
-
-            result
-        })
-        .unwrap()
-}
-
-// Similar to https://doc.rust-lang.org/std/primitive.bool.html#method.then (nightly only),
-// but with a function that returns an `Option<T>` instead of `T`. Adding something like
-// this to std is being discussed: https://github.com/rust-lang/rust/issues/64260
-
-pub trait BoolThen {
-    fn and_then<T>(self, f: impl FnOnce() -> Option<T>) -> Option<T>;
-}
-
-impl BoolThen for bool {
-    fn and_then<T>(self, f: impl FnOnce() -> Option<T>) -> Option<T> {
-        if self {
-            f()
-        } else {
-            None
-        }
-    }
-}
-
-pub fn create_socket(addr: &SocketAddr) -> Socket {
-    let domain = match &addr {
-        SocketAddr::V4(_) => Domain::IPV4,
-        SocketAddr::V6(_) => Domain::IPV6,
-    };
-    let socket =
-        Socket::new(domain, Type::STREAM, Some(Protocol::TCP)).expect("creating socket failed");
-
-    #[cfg(unix)]
-    socket
-        .set_reuse_port(true)
-        .expect("cannot enable SO_REUSEPORT");
-
-    socket.bind(&(*addr).into()).expect("cannot bind");
-
-    socket
-}
-
-/// A module used for serde serialization of bytes in hexadecimal format.
-///
-/// The module is compatible with the serde attribute.
-///
-/// Copied from https://github.com/rust-bitcoin/rust-bitcoincore-rpc/blob/master/json/src/lib.rs
-pub mod serde_hex {
-    use bitcoin::hashes::hex::FromHex;
-    use serde::de::Error;
-    use serde::{Deserializer, Serializer};
-
-    pub fn serialize<S: Serializer>(b: &[u8], s: S) -> Result<S::Ok, S::Error> {
-        s.serialize_str(&hex::encode(b))
-    }
-
-    pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
-        let hex_str: String = ::serde::Deserialize::deserialize(d)?;
-        FromHex::from_hex(&hex_str).map_err(D::Error::custom)
-    }
-
-    pub mod opt {
-        use bitcoin::hashes::hex::FromHex;
-        use serde::de::Error;
-        use serde::{Deserializer, Serializer};
-
-        pub fn serialize<S: Serializer>(b: &Option<Vec<u8>>, s: S) -> Result<S::Ok, S::Error> {
-            match *b {
-                None => s.serialize_none(),
-                Some(ref b) => s.serialize_str(&hex::encode(b)),
-            }
-        }
-
-        pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Vec<u8>>, D::Error> {
-            let hex_str: String = ::serde::Deserialize::deserialize(d)?;
-            Ok(Some(FromHex::from_hex(&hex_str).map_err(D::Error::custom)?))
-        }
-    }
-}

src/util/script.rs

@@ -1,196 +0,0 @@
-#[cfg(feature = "liquid")]
-use elements::address as elements_address;
-
-use crate::chain::{script, Network, Script, TxIn, TxOut};
-use script::Instruction::PushBytes;
-
-pub struct InnerScripts {
-    pub redeem_script: Option<Script>,
-    pub witness_script: Option<Script>,
-}
-
-pub trait IsProvablyUnspendable {
-    fn is_provably_unspendable_(&self) -> bool;
-}
-
-#[cfg(not(feature = "liquid"))]
-impl IsProvablyUnspendable for bitcoin::Script {
-    // is_provably_unspendable() is deprecated in rust-bitcoin
-    // so we re-implement it here. Copy pasted.
-    fn is_provably_unspendable_(&self) -> bool {
-        use bitcoin::blockdata::opcodes::{
-            Class::{IllegalOp, ReturnOp},
-            ClassifyContext, Opcode,
-        };
-
-        match self.as_bytes().first() {
-            Some(b) => {
-                let first = Opcode::from(*b);
-                let class = first.classify(ClassifyContext::Legacy);
-
-                class == ReturnOp || class == IllegalOp
-            }
-            None => false,
-        }
-    }
-}
-
-#[cfg(feature = "liquid")]
-impl IsProvablyUnspendable for elements::Script {
-    #[inline(always)]
-    fn is_provably_unspendable_(&self) -> bool {
-        // Not deprecated yet
-        self.is_provably_unspendable()
-    }
-}
-
-// Extension trait for segwit script detection that works across bitcoin and elements
-pub trait SegwitDetection {
-    fn segwit_is_p2wpkh(&self) -> bool;
-    fn segwit_is_p2wsh(&self) -> bool;
-    fn segwit_is_p2tr(&self) -> bool;
-}
-
-#[cfg(not(feature = "liquid"))]
-impl SegwitDetection for bitcoin::Script {
-    fn segwit_is_p2wpkh(&self) -> bool {
-        self.is_p2wpkh()
-    }
-    fn segwit_is_p2wsh(&self) -> bool {
-        self.is_p2wsh()
-    }
-    fn segwit_is_p2tr(&self) -> bool {
-        self.is_p2tr()
-    }
-}
-
-#[cfg(not(feature = "liquid"))]
-impl SegwitDetection for bitcoin::ScriptBuf {
-    fn segwit_is_p2wpkh(&self) -> bool {
-        self.is_p2wpkh()
-    }
-    fn segwit_is_p2wsh(&self) -> bool {
-        self.is_p2wsh()
-    }
-    fn segwit_is_p2tr(&self) -> bool {
-        self.is_p2tr()
-    }
-}
-
-#[cfg(feature = "liquid")]
-impl SegwitDetection for elements::Script {
-    fn segwit_is_p2wpkh(&self) -> bool {
-        self.is_v0_p2wpkh()
-    }
-    fn segwit_is_p2wsh(&self) -> bool {
-        self.is_v0_p2wsh()
-    }
-    fn segwit_is_p2tr(&self) -> bool {
-        self.is_v1_p2tr()
-    }
-}
-
-pub trait ScriptToAsm: std::fmt::Debug {
-    fn to_asm(&self) -> String {
-        let asm = format!("{:?}", self);
-        asm[7..asm.len() - 1].to_string()
-    }
-}
-impl ScriptToAsm for bitcoin::Script {}
-impl ScriptToAsm for bitcoin::ScriptBuf {}
-#[cfg(feature = "liquid")]
-impl ScriptToAsm for elements::Script {}
-
-pub trait ScriptToAddr {
-    fn to_address_str(&self, network: Network) -> Option<String>;
-}
-#[cfg(not(feature = "liquid"))]
-impl ScriptToAddr for bitcoin::Script {
-    fn to_address_str(&self, network: Network) -> Option<String> {
-        bitcoin::Address::from_script(self, bitcoin::Network::from(network))
-            .ok()
-            .map(|s| s.to_string())
-    }
-}
-#[cfg(feature = "liquid")]
-impl ScriptToAddr for elements::Script {
-    fn to_address_str(&self, network: Network) -> Option<String> {
-        elements_address::Address::from_script(self, None, network.address_params())
-            .map(|a| a.to_string())
-    }
-}
-
-// Returns the witnessScript in the case of p2wsh, or the redeemScript in the case of p2sh.
-pub fn get_innerscripts(txin: &TxIn, prevout: &TxOut) -> InnerScripts {
-    // Wrapped redeemScript for P2SH spends
-    let redeem_script = if prevout.script_pubkey.is_p2sh() {
-        if let Some(Ok(PushBytes(redeemscript))) = txin.script_sig.instructions().last() {
-            #[cfg(not(feature = "liquid"))]
-            let bytes = redeemscript.as_bytes().to_vec();
-            #[cfg(feature = "liquid")]
-            let bytes = redeemscript.to_vec();
-            Some(Script::from(bytes))
-        } else {
-            None
-        }
-    } else {
-        None
-    };
-
-    // Wrapped witnessScript for P2WSH or P2SH-P2WSH spends
-    let witness_script = if prevout.script_pubkey.segwit_is_p2wsh()
-        || prevout.script_pubkey.segwit_is_p2tr()
-        || redeem_script.as_ref().is_some_and(|s| s.segwit_is_p2wsh())
-    {
-        let witness = &txin.witness;
-        #[cfg(feature = "liquid")]
-        let witness = &witness.script_witness;
-
-        // rust-bitcoin returns witness items as a [u8] slice, while rust-elements returns a Vec<u8>
-        #[cfg(not(feature = "liquid"))]
-        let wit_to_vec = Vec::from;
-        #[cfg(feature = "liquid")]
-        let wit_to_vec = Clone::clone;
-
-        let inner_script_slice = if prevout.script_pubkey.segwit_is_p2tr() {
-            // Witness stack is potentially very large
-            // so we avoid to_vec() or iter().collect() for performance
-            let w_len = witness.len();
-            witness
-                .last()
-                // Get the position of the script spend script (if it exists)
-                .map(|last_elem| {
-                    // From BIP341:
-                    // If there are at least two witness elements, and the first byte of
-                    // the last element is 0x50, this last element is called annex a
-                    // and is removed from the witness stack.
-                    if w_len >= 2 && last_elem.first().filter(|&&v| v == 0x50).is_some() {
-                        // account for the extra item removed from the end
-                        3
-                    } else {
-                        // otherwise script is 2nd from last
-                        2
-                    }
-                })
-                // Convert to None if not script spend
-                // Note: Option doesn't have filter_map() method
-                .filter(|&script_pos_from_last| w_len >= script_pos_from_last)
-                .and_then(|script_pos_from_last| {
-                    // Can't use second_to_last() since it might be 3rd to last
-                    #[allow(clippy::iter_nth)]
-                    witness.iter().nth(w_len - script_pos_from_last)
-                })
-        } else {
-            witness.last()
-        };
-
-        inner_script_slice.map(wit_to_vec).map(Script::from)
-    } else {
-        None
-    };
-
-    InnerScripts {
-        redeem_script,
-        witness_script,
-    }
-}

src/util/transaction.rs

@@ -1,392 +0,0 @@
-use crate::chain::{BlockHash, OutPoint, Transaction, TxIn, TxOut, Txid};
-use crate::errors;
-use crate::util::{BlockId, IsProvablyUnspendable};
-
-use std::collections::HashMap;
-
-#[cfg(feature = "liquid")]
-lazy_static! {
-    static ref REGTEST_INITIAL_ISSUANCE_PREVOUT: Txid =
-        "50cdc410c9d0d61eeacc531f52d2c70af741da33af127c364e52ac1ee7c030a5"
-            .parse()
-            .unwrap();
-    static ref TESTNET_INITIAL_ISSUANCE_PREVOUT: Txid =
-        "0c52d2526a5c9f00e9fb74afd15dd3caaf17c823159a514f929ae25193a43a52"
-            .parse()
-            .unwrap();
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct TransactionStatus {
-    pub confirmed: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub block_height: Option<usize>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub block_hash: Option<BlockHash>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub block_time: Option<u32>,
-}
-
-impl From<Option<BlockId>> for TransactionStatus {
-    fn from(blockid: Option<BlockId>) -> TransactionStatus {
-        match blockid {
-            Some(b) => TransactionStatus {
-                confirmed: true,
-                block_height: Some(b.height),
-                block_hash: Some(b.hash),
-                block_time: Some(b.time),
-            },
-            None => TransactionStatus {
-                confirmed: false,
-                block_height: None,
-                block_hash: None,
-                block_time: None,
-            },
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct TxInput {
-    pub txid: Txid,
-    pub vin: u32,
-}
-
-pub fn is_coinbase(txin: &TxIn) -> bool {
-    #[cfg(not(feature = "liquid"))]
-    return txin.previous_output.is_null();
-    #[cfg(feature = "liquid")]
-    return txin.is_coinbase();
-}
-
-pub fn has_prevout(txin: &TxIn) -> bool {
-    #[cfg(not(feature = "liquid"))]
-    return !txin.previous_output.is_null();
-    #[cfg(feature = "liquid")]
-    return !txin.is_coinbase()
-        && !txin.is_pegin
-        && txin.previous_output.txid != *REGTEST_INITIAL_ISSUANCE_PREVOUT
-        && txin.previous_output.txid != *TESTNET_INITIAL_ISSUANCE_PREVOUT;
-}
-
-pub fn is_spendable(txout: &TxOut) -> bool {
-    #[cfg(not(feature = "liquid"))]
-    return !txout.script_pubkey.is_provably_unspendable_();
-    #[cfg(feature = "liquid")]
-    return !txout.is_fee() && !txout.script_pubkey.is_provably_unspendable_();
-}
-
-/// Extract the previous TxOuts of a Transaction's TxIns
-///
-/// # Errors
-///
-/// This function MUST NOT return an error variant when allow_missing is true.
-/// If allow_missing is false, it will return an error when any Outpoint is
-/// missing from the keys of the txos argument's HashMap.
-pub fn extract_tx_prevouts<'a>(
-    tx: &Transaction,
-    txos: &'a HashMap<OutPoint, TxOut>,
-) -> Result<HashMap<u32, &'a TxOut>, errors::Error> {
-    tx.input
-        .iter()
-        .enumerate()
-        .filter(|(_, txi)| has_prevout(txi))
-        .map(|(index, txi)| {
-            Ok((
-                index as u32,
-                match txos.get(&txi.previous_output) {
-                    Some(txo) => txo,
-                    None => {
-                        return Err(format!("missing outpoint {:?}", txi.previous_output).into());
-                    }
-                },
-            ))
-        })
-        .collect()
-}
-
-pub fn serialize_outpoint<S>(outpoint: &OutPoint, serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: serde::ser::Serializer,
-{
-    use serde::ser::SerializeStruct;
-    let mut s = serializer.serialize_struct("OutPoint", 2)?;
-    s.serialize_field("txid", &outpoint.txid)?;
-    s.serialize_field("vout", &outpoint.vout)?;
-    s.end()
-}
-
-pub(super) mod sigops {
-    use crate::chain::{
-        opcodes::all::{OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY},
-        script::{self, Instruction},
-        Transaction, TxOut, Witness,
-    };
-    #[cfg(not(feature = "liquid"))]
-    use bitcoin::opcodes::Opcode;
-    #[cfg(feature = "liquid")]
-    use elements::opcodes::All as Opcode;
-    use std::collections::HashMap;
-
-    /// Get sigop count for transaction. prevout_map must have all the prevouts.
-    pub fn transaction_sigop_count(
-        tx: &Transaction,
-        prevout_map: &HashMap<u32, &TxOut>,
-    ) -> Result<usize, script::Error> {
-        let input_count = tx.input.len();
-        let mut prevouts = Vec::with_capacity(input_count);
-
-        #[cfg(not(feature = "liquid"))]
-        let is_coinbase_or_pegin = tx.is_coinbase();
-        #[cfg(feature = "liquid")]
-        let is_coinbase_or_pegin = tx.is_coinbase() || tx.input.iter().any(|input| input.is_pegin);
-
-        if !is_coinbase_or_pegin {
-            for idx in 0..input_count {
-                prevouts.push(
-                    *prevout_map
-                        .get(&(idx as u32))
-                        .ok_or(script::Error::EarlyEndOfScript)?,
-                );
-            }
-        }
-
-        // coinbase tx won't use prevouts so it can be empty.
-        get_sigop_cost(tx, &prevouts, true, true)
-    }
-
-    fn decode_pushnum(op: &Opcode) -> Option<u8> {
-        // 81 = OP_1, 96 = OP_16
-        // 81 -> 1, so... 81 - 80 -> 1
-        #[cfg(not(feature = "liquid"))]
-        let self_u8 = op.to_u8();
-        #[cfg(feature = "liquid")]
-        let self_u8 = op.into_u8();
-        match self_u8 {
-            81..=96 => Some(self_u8 - 80),
-            _ => None,
-        }
-    }
-
-    fn count_sigops(script: &script::Script, accurate: bool) -> usize {
-        let mut n = 0;
-        let mut pushnum_cache = None;
-        for inst in script.instructions() {
-            match inst {
-                Ok(Instruction::Op(opcode)) => {
-                    match opcode {
-                        OP_CHECKSIG | OP_CHECKSIGVERIFY => {
-                            n += 1;
-                        }
-                        OP_CHECKMULTISIG | OP_CHECKMULTISIGVERIFY => {
-                            match (accurate, pushnum_cache) {
-                                (true, Some(pushnum)) => {
-                                    // Add the number of pubkeys in the multisig as sigop count
-                                    n += usize::from(pushnum);
-                                }
-                                _ => {
-                                    // MAX_PUBKEYS_PER_MULTISIG from Bitcoin Core
-                                    // https://github.com/bitcoin/bitcoin/blob/v25.0/src/script/script.h#L29-L30
-                                    n += 20;
-                                }
-                            }
-                        }
-                        _ => {
-                            pushnum_cache = decode_pushnum(&opcode);
-                        }
-                    }
-                }
-                // We ignore errors as well as pushdatas
-                _ => {
-                    pushnum_cache = None;
-                }
-            }
-        }
-
-        n
-    }
-
-    /// Get the sigop count for legacy transactions
-    fn get_legacy_sigop_count(tx: &Transaction) -> usize {
-        let mut n = 0;
-        for input in &tx.input {
-            n += count_sigops(&input.script_sig, false);
-        }
-        for output in &tx.output {
-            n += count_sigops(&output.script_pubkey, false);
-        }
-        n
-    }
-
-    fn get_p2sh_sigop_count(tx: &Transaction, previous_outputs: &[&TxOut]) -> usize {
-        #[cfg(not(feature = "liquid"))]
-        if tx.is_coinbase() {
-            return 0;
-        }
-        #[cfg(feature = "liquid")]
-        if tx.is_coinbase() {
-            return 0;
-        }
-        let mut n = 0;
-        for (input, prevout) in tx.input.iter().zip(previous_outputs.iter()) {
-            if prevout.script_pubkey.is_p2sh() {
-                if let Some(Ok(script::Instruction::PushBytes(redeem))) =
-                    input.script_sig.instructions().last()
-                {
-                    #[cfg(not(feature = "liquid"))]
-                    let script = script::Script::from_bytes(redeem.as_bytes());
-                    #[cfg(feature = "liquid")]
-                    let script = script::Script::from(redeem.to_vec());
-                    #[allow(clippy::needless_borrow)]
-                    {
-                        n += count_sigops(&script, true);
-                    }
-                }
-            }
-        }
-        n
-    }
-
-    fn get_witness_sigop_count(tx: &Transaction, previous_outputs: &[&TxOut]) -> usize {
-        let mut n = 0;
-
-        #[inline]
-        fn is_push_only(script: &script::Script) -> bool {
-            for inst in script.instructions() {
-                match inst {
-                    Err(_) => return false,
-                    Ok(Instruction::Op(_)) => return false,
-                    Ok(Instruction::PushBytes(_)) => {}
-                }
-            }
-            true
-        }
-
-        #[inline]
-        fn last_pushdata(script: &script::Script) -> Option<&[u8]> {
-            match script.instructions().last() {
-                #[cfg(not(feature = "liquid"))]
-                Some(Ok(Instruction::PushBytes(bytes))) => Some(bytes.as_bytes()),
-                #[cfg(feature = "liquid")]
-                Some(Ok(Instruction::PushBytes(bytes))) => Some(bytes),
-                _ => None,
-            }
-        }
-
-        #[inline]
-        fn count_with_prevout(
-            prevout: &TxOut,
-            script_sig: &script::Script,
-            witness: &Witness,
-        ) -> usize {
-            let mut n = 0;
-
-            let script_owned;
-            let script: &script::Script = if prevout.script_pubkey.is_witness_program() {
-                &prevout.script_pubkey
-            } else if prevout.script_pubkey.is_p2sh()
-                && is_push_only(script_sig)
-                && !script_sig.is_empty()
-            {
-                #[cfg(not(feature = "liquid"))]
-                {
-                    script_owned =
-                        script::ScriptBuf::from(last_pushdata(script_sig).unwrap().to_vec());
-                }
-                #[cfg(feature = "liquid")]
-                {
-                    script_owned =
-                        script::Script::from(last_pushdata(script_sig).unwrap().to_vec());
-                }
-                &script_owned
-            } else {
-                return 0;
-            };
-
-            #[cfg(not(feature = "liquid"))]
-            if script.is_p2wsh() {
-                let bytes = script.as_bytes();
-                n += sig_ops(witness, bytes[0], &bytes[2..]);
-            } else if script.is_p2wpkh() {
-                n += 1;
-            }
-            #[cfg(feature = "liquid")]
-            if script.is_v0_p2wsh() {
-                let bytes = script.as_bytes();
-                n += sig_ops(witness, bytes[0], &bytes[2..]);
-            } else if script.is_v0_p2wpkh() {
-                n += 1;
-            }
-            n
-        }
-
-        for (input, prevout) in tx.input.iter().zip(previous_outputs.iter()) {
-            n += count_with_prevout(prevout, &input.script_sig, &input.witness);
-        }
-        n
-    }
-
-    /// Get the sigop cost for this transaction.
-    fn get_sigop_cost(
-        tx: &Transaction,
-        previous_outputs: &[&TxOut],
-        verify_p2sh: bool,
-        verify_witness: bool,
-    ) -> Result<usize, script::Error> {
-        let mut n_sigop_cost = get_legacy_sigop_count(tx) * 4;
-        #[cfg(not(feature = "liquid"))]
-        if tx.is_coinbase() {
-            return Ok(n_sigop_cost);
-        }
-        #[cfg(feature = "liquid")]
-        if tx.is_coinbase() || tx.input.iter().any(|input| input.is_pegin) {
-            return Ok(n_sigop_cost);
-        }
-        if tx.input.len() != previous_outputs.len() {
-            return Err(script::Error::EarlyEndOfScript);
-        }
-        if verify_witness && !verify_p2sh {
-            return Err(script::Error::EarlyEndOfScript);
-        }
-        if verify_p2sh {
-            n_sigop_cost += get_p2sh_sigop_count(tx, previous_outputs) * 4;
-        }
-        if verify_witness {
-            n_sigop_cost += get_witness_sigop_count(tx, previous_outputs);
-        }
-
-        Ok(n_sigop_cost)
-    }
-
-    /// Get sigops for the Witness
-    ///
-    /// witness_version is the raw opcode. OP_0 is 0, OP_1 is 81, etc.
-    #[allow(clippy::redundant_closure)]
-    fn sig_ops(witness: &Witness, witness_version: u8, witness_program: &[u8]) -> usize {
-        #[cfg(feature = "liquid")]
-        let last_witness = witness.script_witness.last();
-        #[cfg(not(feature = "liquid"))]
-        let last_witness = witness.last();
-        match (witness_version, witness_program.len()) {
-            (0, 20) => 1,
-            (0, 32) => {
-                #[cfg(not(feature = "liquid"))]
-                {
-                    #[allow(clippy::needless_borrow)]
-                    last_witness
-                        .map(|sl| script::Script::from_bytes(sl))
-                        .map(|s| count_sigops(s, true))
-                        .unwrap_or_default()
-                }
-                #[cfg(feature = "liquid")]
-                {
-                    last_witness
-                        .map(|sl| script::Script::from(sl.clone()))
-                        .map(|s| count_sigops(&s, true))
-                        .unwrap_or_default()
-                }
-            }
-            _ => 0,
-        }
-    }
-}

src/utils/address.rs

@@ -0,0 +1,349 @@
+// Rust Bitcoin Library
+// Written in 2014 by
+//     Andrew Poelstra <apoelstra@wpsoftware.net>
+// To the extent possible under law, the author(s) have dedicated all
+// copyright and related and neighboring rights to this software to
+// the public domain worldwide. This software is distributed without
+// any warranty.
+//
+// You should have received a copy of the CC0 Public Domain Dedication
+// along with this software.
+// If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
+//
+
+//! Addresses
+//!
+//! Support for ordinary base58 Bitcoin addresses and private keys
+//!
+
+use std::fmt::{self, Display, Formatter};
+use std::str::FromStr;
+
+use bitcoin_bech32::{self, u5, WitnessProgram};
+use secp256k1::key::PublicKey;
+
+use bitcoin::blockdata::opcodes;
+use bitcoin::blockdata::script;
+use bitcoin::consensus::encode;
+use bitcoin::util::base58;
+use bitcoin::util::hash::Hash160;
+use daemon::Network;
+
+/// The method used to produce an address
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Payload {
+    /// pay-to-pubkey
+    Pubkey(PublicKey),
+    /// pay-to-pkhash address
+    PubkeyHash(Hash160),
+    /// P2SH address
+    ScriptHash(Hash160),
+    /// Segwit address
+    WitnessProgram(WitnessProgram),
+}
+
+// Originally was: #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Clone, PartialEq, Hash)]
+/// A Bitcoin address
+pub struct Address {
+    /// The type of the address
+    pub payload: Payload,
+    /// The network on which this address is usable
+    pub network: Network,
+}
+
+impl Address {
+    /// Creates a pay to (compressed) public key hash address from a public key
+    /// This is the preferred non-witness type address
+    #[inline]
+    pub fn p2pkh(pk: &PublicKey, network: Network) -> Address {
+        Address {
+            network: network,
+            payload: Payload::PubkeyHash(Hash160::from_data(&pk.serialize()[..])),
+        }
+    }
+
+    /// Creates a pay to uncompressed public key hash address from a public key
+    /// This address type is discouraged as it uses more space but otherwise equivalent to p2pkh
+    /// therefore only adds ambiguity
+    #[inline]
+    pub fn p2upkh(pk: &PublicKey, network: Network) -> Address {
+        Address {
+            network: network,
+            payload: Payload::PubkeyHash(Hash160::from_data(&pk.serialize_uncompressed()[..])),
+        }
+    }
+
+    /// Creates a pay to public key address from a public key
+    /// This address type was used in the early history of Bitcoin.
+    /// Satoshi's coins are still on addresses of this type.
+    #[inline]
+    pub fn p2pk(pk: &PublicKey, network: Network) -> Address {
+        Address {
+            network: network,
+            payload: Payload::Pubkey(*pk),
+        }
+    }
+
+    /// Creates a pay to script hash P2SH address from a script
+    /// This address type was introduced with BIP16 and is the popular ty implement multi-sig these days.
+    #[inline]
+    pub fn p2sh(script: &script::Script, network: Network) -> Address {
+        Address {
+            network: network,
+            payload: Payload::ScriptHash(Hash160::from_data(&script[..])),
+        }
+    }
+
+    /// Create a witness pay to public key address from a public key
+    /// This is the native segwit address type for an output redemable with a single signature
+    pub fn p2wpkh(pk: &PublicKey, network: Network) -> Address {
+        Address {
+            network: network,
+            payload: Payload::WitnessProgram(
+                // unwrap is safe as witness program is known to be correct as above
+                WitnessProgram::new(
+                    u5::try_from_u8(0).expect("0<32"),
+                    Hash160::from_data(&pk.serialize()[..])[..].to_vec(),
+                    Address::bech_network(network),
+                )
+                .unwrap(),
+            ),
+        }
+    }
+
+    /// Create a pay to script address that embeds a witness pay to public key
+    /// This is a segwit address type that looks familiar (as p2sh) to legacy clients
+    pub fn p2shwpkh(pk: &PublicKey, network: Network) -> Address {
+        let builder = script::Builder::new()
+            .push_int(0)
+            .push_slice(&Hash160::from_data(&pk.serialize()[..])[..]);
+        Address {
+            network: network,
+            payload: Payload::ScriptHash(Hash160::from_data(builder.into_script().as_bytes())),
+        }
+    }
+
+    /// Create a witness pay to script hash address
+    pub fn p2wsh(script: &script::Script, network: Network) -> Address {
+        use crypto::digest::Digest;
+        use crypto::sha2::Sha256;
+
+        let mut digest = Sha256::new();
+        digest.input(script.as_bytes());
+        let mut d = [0u8; 32];
+        digest.result(&mut d);
+
+        Address {
+            network: network,
+            payload: Payload::WitnessProgram(
+                // unwrap is safe as witness program is known to be correct as above
+                WitnessProgram::new(
+                    u5::try_from_u8(0).expect("0<32"),
+                    d.to_vec(),
+                    Address::bech_network(network),
+                )
+                .unwrap(),
+            ),
+        }
+    }
+
+    /// Create a pay to script address that embeds a witness pay to script hash address
+    /// This is a segwit address type that looks familiar (as p2sh) to legacy clients
+    pub fn p2shwsh(script: &script::Script, network: Network) -> Address {
+        use crypto::digest::Digest;
+        use crypto::sha2::Sha256;
+
+        let mut digest = Sha256::new();
+        digest.input(script.as_bytes());
+        let mut d = [0u8; 32];
+        digest.result(&mut d);
+        let ws = script::Builder::new()
+            .push_int(0)
+            .push_slice(&d)
+            .into_script();
+
+        Address {
+            network: network,
+            payload: Payload::ScriptHash(Hash160::from_data(ws.as_bytes())),
+        }
+    }
+
+    #[inline]
+    /// convert Network to bech32 network (this should go away soon)
+    fn bech_network(network: Network) -> bitcoin_bech32::constants::Network {
+        match network {
+            Network::Bitcoin => bitcoin_bech32::constants::Network::Bitcoin,
+            Network::Testnet => bitcoin_bech32::constants::Network::Testnet,
+            Network::Regtest => bitcoin_bech32::constants::Network::Regtest,
+            // this should never actually happen, Liquid does not have bech32 addresses
+            Network::Liquid | Network::LiquidV1 | Network::LiquidRegtest => {
+                bitcoin_bech32::constants::Network::Bitcoin
+            }
+        }
+    }
+
+    /// Generates a script pubkey spending to this address
+    pub fn script_pubkey(&self) -> script::Script {
+        match self.payload {
+            Payload::Pubkey(ref pk) => script::Builder::new()
+                .push_slice(&pk.serialize_uncompressed()[..])
+                .push_opcode(opcodes::All::OP_CHECKSIG),
+            Payload::PubkeyHash(ref hash) => script::Builder::new()
+                .push_opcode(opcodes::All::OP_DUP)
+                .push_opcode(opcodes::All::OP_HASH160)
+                .push_slice(&hash[..])
+                .push_opcode(opcodes::All::OP_EQUALVERIFY)
+                .push_opcode(opcodes::All::OP_CHECKSIG),
+            Payload::ScriptHash(ref hash) => script::Builder::new()
+                .push_opcode(opcodes::All::OP_HASH160)
+                .push_slice(&hash[..])
+                .push_opcode(opcodes::All::OP_EQUAL),
+            Payload::WitnessProgram(ref witprog) => script::Builder::new()
+                .push_int(witprog.version().to_u8() as i64)
+                .push_slice(witprog.program()),
+        }
+        .into_script()
+    }
+}
+
+impl Display for Address {
+    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
+        match self.payload {
+            // note: serialization for pay-to-pk is defined, but is irreversible
+            Payload::Pubkey(ref pk) => {
+                let hash = &Hash160::from_data(&pk.serialize_uncompressed()[..]);
+                let mut prefixed = [0; 21];
+                prefixed[0] = match self.network {
+                    Network::Bitcoin => 0,
+                    Network::Testnet | Network::Regtest => 111,
+                    Network::Liquid | Network::LiquidV1 => 57,
+                    Network::LiquidRegtest => 196,
+                };
+                prefixed[1..].copy_from_slice(&hash[..]);
+                base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
+            }
+            Payload::PubkeyHash(ref hash) => {
+                let mut prefixed = [0; 21];
+                prefixed[0] = match self.network {
+                    Network::Bitcoin => 0,
+                    Network::Testnet | Network::Regtest => 111,
+                    Network::Liquid | Network::LiquidV1 => 57,
+                    Network::LiquidRegtest => 235,
+                };
+                prefixed[1..].copy_from_slice(&hash[..]);
+                base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
+            }
+            Payload::ScriptHash(ref hash) => {
+                let mut prefixed = [0; 21];
+                prefixed[0] = match self.network {
+                    Network::Bitcoin => 5,
+                    Network::Testnet | Network::Regtest => 196,
+                    Network::Liquid | Network::LiquidV1 => 39,
+                    Network::LiquidRegtest => 75,
+                };
+                prefixed[1..].copy_from_slice(&hash[..]);
+                base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
+            }
+            Payload::WitnessProgram(ref witprog) => fmt.write_str(&witprog.to_address()),
+        }
+    }
+}
+
+impl FromStr for Address {
+    type Err = encode::Error;
+
+    fn from_str(s: &str) -> Result<Address, encode::Error> {
+        // bech32 (note that upper or lowercase is allowed but NOT mixed case)
+        if s.starts_with("bc1")
+            || s.starts_with("BC1")
+            || s.starts_with("tb1")
+            || s.starts_with("TB1")
+            || s.starts_with("bcrt1")
+            || s.starts_with("BCRT1")
+        {
+            let witprog = WitnessProgram::from_address(s)?;
+            let network = match witprog.network() {
+                bitcoin_bech32::constants::Network::Bitcoin => Network::Bitcoin,
+                bitcoin_bech32::constants::Network::Testnet => Network::Testnet,
+                bitcoin_bech32::constants::Network::Regtest => Network::Regtest,
+                _ => panic!("unknown network"),
+            };
+            if witprog.version().to_u8() != 0 {
+                return Err(encode::Error::UnsupportedWitnessVersion(
+                    witprog.version().to_u8(),
+                ));
+            }
+            return Ok(Address {
+                network: network,
+                payload: Payload::WitnessProgram(witprog),
+            });
+        }
+
+        if s.len() > 50 {
+            return Err(encode::Error::Base58(base58::Error::InvalidLength(
+                s.len() * 11 / 15,
+            )));
+        }
+
+        // Base 58
+        let data = base58::from_check(s)?;
+
+        if data.len() != 21 {
+            return Err(encode::Error::Base58(base58::Error::InvalidLength(
+                data.len(),
+            )));
+        }
+
+        let (network, payload) = match data[0] {
+            0 => (
+                Network::Bitcoin,
+                Payload::PubkeyHash(Hash160::from(&data[1..])),
+            ),
+            5 => (
+                Network::Bitcoin,
+                Payload::ScriptHash(Hash160::from(&data[1..])),
+            ),
+            111 => (
+                Network::Testnet,
+                Payload::PubkeyHash(Hash160::from(&data[1..])),
+            ),
+            196 => (
+                Network::Testnet,
+                Payload::ScriptHash(Hash160::from(&data[1..])),
+            ),
+            57 => (
+                Network::LiquidV1,
+                Payload::PubkeyHash(Hash160::from(&data[1..])),
+            ),
+            39 => (
+                Network::LiquidV1,
+                Payload::ScriptHash(Hash160::from(&data[1..])),
+            ),
+            235 => (
+                Network::LiquidRegtest,
+                Payload::PubkeyHash(Hash160::from(&data[1..])),
+            ),
+            75 => (
+                Network::LiquidRegtest,
+                Payload::ScriptHash(Hash160::from(&data[1..])),
+            ),
+            x => {
+                return Err(encode::Error::Base58(base58::Error::InvalidVersion(vec![
+                    x,
+                ])))
+            }
+        };
+
+        Ok(Address {
+            network: network,
+            payload: payload,
+        })
+    }
+}
+
+impl ::std::fmt::Debug for Address {
+    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}

src/utils/mod.rs

@@ -0,0 +1 @@
+pub mod address;

start

@@ -1,274 +0,0 @@
-#!/usr/bin/env zsh
-
-# initialize variables
-DAEMON=bitcoin
-NETWORK=mainnet
-FEATURES=default
-DB_FOLDER=/electrs
-ASSET_DB_ARGS=()
-NODENAME=$(hostname|cut -d . -f1)
-LOCATION=$(hostname|cut -d . -f2)
-USAGE="Usage: $0 (mainnet|testnet|signet|liquid|liquidtestnet) [popular-scripts]"
-
-# load rust if necessary
-if [ -e "${HOME}/.cargo/env" ];then
-	source "${HOME}/.cargo/env"
-	export PATH="${HOME}/.cargo/bin:${PATH}"
-fi
-
-# which OS?
-case "$(uname -s)" in
-	FreeBSD)
-		OS=FreeBSD
-		NPROC=$(sysctl -n hw.ncpu)
-		export CC=/usr/local/bin/clang17
-		export CXX=/usr/local/bin/clang++17
-		export CPP=/usr/local/bin/clang-cpp17
-		export RUSTFLAGS="-C linker=clang17"
-	;;
-	Darwin)
-		OS=Darwin
-		NPROC=$(sysctl -n hw.ncpu)
-	;;
-	Linux)
-		OS=Linux
-		NPROC=$(grep -c proc /proc/cpuinfo)
-	;;
-	*)
-		OS=Unknown
-		NPROC=4
-	;;
-esac
-
-# which network?
-case "${1}" in
-	mainnet)
-		THREADS=$((NPROC / 8))
-		CRONJOB_TIMING="20 4 * * *"
-	;;
-	testnet)
-		NETWORK=testnet
-		THREADS=$((NPROC / 8))
-		CRONJOB_TIMING="2 4 * * *"
-	;;
-	testnet4)
-		NETWORK=testnet4
-		THREADS=$((NPROC / 8))
-		CRONJOB_TIMING="17 4 * * *"
-	;;
-	signet)
-		NETWORK=signet
-		THREADS=$((NPROC / 8))
-		CRONJOB_TIMING="9 4 * * *"
-	;;
-	liquid)
-		DAEMON=elements
-		NETWORK=liquid
-		FEATURES=liquid
-		ASSET_DB_ARGS=(--asset-db-path /elements/asset_registry_db)
-		THREADS=$((NPROC / 8))
-		CRONJOB_TIMING="12 4 * * *"
-	;;
-	liquidtestnet)
-		DAEMON=elements
-		NETWORK=liquidtestnet
-		FEATURES=liquid
-		ASSET_DB_ARGS=(--asset-db-path /elements/asset_registry_testnet_db)
-		THREADS=$((NPROC / 8))
-		CRONJOB_TIMING="17 4 * * *"
-	;;
-	*)
-		echo "${USAGE}"
-		exit 1
-	;;
-esac
-
-# Run the popular address txt file generator before each run
-POPULAR_SCRIPTS_FOLDER="${HOME}/popular-scripts/${NETWORK}"
-POPULAR_SCRIPTS_FILE_RAW="${POPULAR_SCRIPTS_FOLDER}/popular-scripts-raw.txt"
-POPULAR_SCRIPTS_FILE="${POPULAR_SCRIPTS_FOLDER}/popular-scripts.txt"
-
-# This function runs the job for generating the popular scripts text file for the precache arg
-generate_popular_scripts() {
-	mkdir -p "${POPULAR_SCRIPTS_FOLDER}"
-
-	## Use nproc * 4 threads to generate the txt file (lots of iowait, so 2x~4x core count is ok)
-	## Only pick up addresses with 101 history events or more
-	## (Without lowering MIN_HISTORY_ITEMS_TO_CACHE this is the lowest we can go)
-	## It prints out progress to STDERR
-	echo "[*] Generating popular-scripts using ${THREADS} threads..."
-	cd "${HOME}/electrs"
-	HIGH_USAGE_THRESHOLD=101 \
-	JOB_THREAD_COUNT=${THREADS} \
-	nice cargo run \
-		--release \
-		--bin popular-scripts \
-		--features "${FEATURES}" \
-		-- \
-		--network "${NETWORK}" \
-		--db-dir "${DB_FOLDER}" \
-		> "${POPULAR_SCRIPTS_FILE_RAW}"
-
-	## Only overwrite the existing file if the popular-scripts cargo run succeeded
-	if [ "$?" = "0" ];then
-		## Sorted and deduplicated just in case
-		echo "Sorting popular scripts for final results..."
-		sort "${POPULAR_SCRIPTS_FILE_RAW}" | uniq > "${POPULAR_SCRIPTS_FILE}"
-	fi
-
-	rm "${POPULAR_SCRIPTS_FILE_RAW}"
-}
-
-# This function is for inserting the cronjob for generating the popular scripts
-CRONJOB_CMD="\"${HOME}/electrs/start\" \"${NETWORK}\" popular-scripts"
-echo "${CRONJOB_TIMING} ${CRONJOB_CMD}"
-
-case "${2}" in
-	popular-scripts)
-		echo "[*] Only generate popular-scripts, then exit"
-		generate_popular_scripts
-		exit 0
-	;;
-	version)
-		echo "[*] Only print versions, then exit"
-		cargo run --bin electrs --release -- --version
-		cargo run --bin popular-scripts --release -- --version
-		exit 0
-	;;
-	"")
-		# If the 2nd arg isn't passed, just run the normal electrs script as-is
-	;;
-	*)
-		echo "${USAGE}"
-		exit 1
-	;;
-esac
-
-# run in loop in case of crash
-until false
-do
-	# reset CWD
-	cd "${HOME}/electrs"
-
-	# disable making electrs.core files
-	ulimit -c 0
-
-	# prepare run-time variables
-	UTXOS_LIMIT=500
-	ELECTRUM_TXS_LIMIT=500
-	ELECTRUM_MAX_LINE_SIZE=1048576 # 1 MiB
-	ELECTRUM_MAX_SUBSCRIPTIONS=100
-	ELECTRUM_MAX_CLIENTS=1000
-	MAIN_LOOP_DELAY=500
-	DAEMON_CONF="${HOME}/${DAEMON}.conf"
-	HTTP_SOCKET_FILE="${HOME}/socket/esplora-${DAEMON}-${NETWORK}"
-	RPC_SOCKET_FILE="${HOME}/socket/electrum-${DAEMON}-${NETWORK}"
-
-	# get RPC credentials from bitcoin.conf or elements.conf directly
-	echo "[*] Getting RPC credentials from ${DAEMON_CONF}"
-	RPC_USER=$(grep 'rpcuser=' "${DAEMON_CONF}"|cut -d = -f2|head -1)
-	RPC_PASS=$(grep 'rpcpassword=' "${DAEMON_CONF}"|cut -d = -f2|head -1)
-
-	# override limits based on hostname
-	if [ "${NODENAME}" = "node201" ];then
-		UTXOS_LIMIT=9000
-		ELECTRUM_TXS_LIMIT=9000
-		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
-		ELECTRUM_MAX_SUBSCRIPTIONS=100000
-		ELECTRUM_MAX_CLIENTS=10000
-		MAIN_LOOP_DELAY=14000
-	fi
-	if [ "${NODENAME}" = "node204" ] && [ "${LOCATION}" = "sg1" ];then
-		UTXOS_LIMIT=9000
-		ELECTRUM_TXS_LIMIT=9000
-		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
-		ELECTRUM_MAX_SUBSCRIPTIONS=100000
-		ELECTRUM_MAX_CLIENTS=10000
-	fi
-	if [ "${NODENAME}" = "node204" ] && [ "${LOCATION}" = "hnl" ];then
-		UTXOS_LIMIT=9000
-		ELECTRUM_TXS_LIMIT=9000
-		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
-		ELECTRUM_MAX_SUBSCRIPTIONS=100000
-		ELECTRUM_MAX_CLIENTS=10000
-	fi
-	if [ "${NODENAME}" = "node206" ] && [ "${LOCATION}" = "tk7" ];then
-		UTXOS_LIMIT=9000
-		ELECTRUM_TXS_LIMIT=9000
-		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
-		ELECTRUM_MAX_SUBSCRIPTIONS=100000
-		ELECTRUM_MAX_CLIENTS=10000
-	fi
-	if [ "${NODENAME}" = "node211" ];then
-		UTXOS_LIMIT=9000
-		ELECTRUM_TXS_LIMIT=9000
-		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
-		ELECTRUM_MAX_SUBSCRIPTIONS=100000
-		ELECTRUM_MAX_CLIENTS=10000
-	fi
-	if [ "${NODENAME}" = "node212" ];then
-		UTXOS_LIMIT=9000
-		ELECTRUM_TXS_LIMIT=9000
-		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
-		ELECTRUM_MAX_SUBSCRIPTIONS=100000
-		ELECTRUM_MAX_CLIENTS=10000
-	fi
-	if [ "${NODENAME}" = "node213" ];then
-		UTXOS_LIMIT=9000
-		ELECTRUM_TXS_LIMIT=9000
-		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
-		ELECTRUM_MAX_SUBSCRIPTIONS=100000
-		ELECTRUM_MAX_CLIENTS=10000
-	fi
-	if [ "${NODENAME}" = "node214" ];then
-		UTXOS_LIMIT=9000
-		ELECTRUM_TXS_LIMIT=9000
-		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
-		ELECTRUM_MAX_SUBSCRIPTIONS=100000
-		ELECTRUM_MAX_CLIENTS=10000
-	fi
-	if [ "${NETWORK}" = "testnet4" ];then
-		UTXOS_LIMIT=9000
-		ELECTRUM_TXS_LIMIT=9000
-		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
-		ELECTRUM_MAX_SUBSCRIPTIONS=100000
-		ELECTRUM_MAX_CLIENTS=10000
-	fi
-	if [ "${LOCATION}" = "fmt" ];then
-		UTXOS_LIMIT=9000
-		ELECTRUM_TXS_LIMIT=9000
-		ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
-		ELECTRUM_MAX_SUBSCRIPTIONS=100000
-		ELECTRUM_MAX_CLIENTS=10000
-	fi
-
-	if [ ! -e "${POPULAR_SCRIPTS_FILE}" ];then
-		generate_popular_scripts
-	fi
-
-	# Run the electrs process (Note: db-dir is used in both commands)
-	nice cargo run \
-		--release \
-		--bin electrs \
-		--features "${FEATURES}" \
-		-- \
-		--network "${NETWORK}" \
-		--daemon-dir "${HOME}" \
-		--db-dir "${DB_FOLDER}" \
-		"${ASSET_DB_ARGS[@]}" \
-		--main-loop-delay "${MAIN_LOOP_DELAY}" \
-		--rpc-socket-file "${RPC_SOCKET_FILE}" \
-		--http-socket-file "${HTTP_SOCKET_FILE}" \
-		--precache-scripts "${POPULAR_SCRIPTS_FILE}" \
-		--precache-threads "${THREADS}" \
-		--cookie "${RPC_USER}:${RPC_PASS}" \
-		--cors '*' \
-		--address-search \
-		--utxos-limit "${UTXOS_LIMIT}" \
-		--electrum-txs-limit "${ELECTRUM_TXS_LIMIT}" \
-		--electrum-max-line-size "${ELECTRUM_MAX_LINE_SIZE}" \
-		--electrum-max-subscriptions "${ELECTRUM_MAX_SUBSCRIPTIONS}" \
-		--electrum-max-clients "${ELECTRUM_MAX_CLIENTS}" \
-		-vv
-	sleep 1
-done