Compare commits
No commits in common. "mempool" and "update_libs" have entirely different histories.
mempool
...
update_lib
@ -1 +0,0 @@
|
||||
Dockerfile
|
||||
@ -1,6 +0,0 @@
|
||||
# see https://editorconfig.org for more options, and setup instructions for yours editor
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
|
||||
50
.github/actions/ci-rust-setup/action.yml
vendored
50
.github/actions/ci-rust-setup/action.yml
vendored
@ -1,50 +0,0 @@
|
||||
name: CI Rust Setup
|
||||
description: 'Sets up the environment for Rust jobs during CI workflow'
|
||||
|
||||
inputs:
|
||||
cache-name:
|
||||
description: 'Name of cache artifacts (same name is same cache key) empty to disable cache'
|
||||
required: false
|
||||
targets:
|
||||
description: 'A comma separated list of extra targets you want to install'
|
||||
required: false
|
||||
components:
|
||||
description: 'A comma separated list of extra components you want to install'
|
||||
required: false
|
||||
toolchain:
|
||||
description: 'The toolchain to use. If not specified, the rust-toolchain file will be used'
|
||||
required: false
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Get toolchain from input OR rust-toolchain file
|
||||
id: gettoolchain
|
||||
shell: bash
|
||||
run: |-
|
||||
RUST_TOOLCHAIN="${{ inputs.toolchain }}"
|
||||
if [ ! -f rust-toolchain ] && [ -z "${RUST_TOOLCHAIN}" ]; then
|
||||
echo "***ERROR*** NEED toolchain INPUT OR rust-toolchain FILE IN ROOT OF REPOSITORY" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${RUST_TOOLCHAIN}" ]; then
|
||||
RUST_TOOLCHAIN="$(cat rust-toolchain)"
|
||||
fi
|
||||
echo "toolchain=\"${RUST_TOOLCHAIN}\"" >> $GITHUB_OUTPUT
|
||||
- name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
|
||||
id: toolchain
|
||||
# Commit date is Nov 18, 2024
|
||||
uses: dtolnay/rust-toolchain@315e265cd78dad1e1dcf3a5074f6d6c47029d5aa
|
||||
with:
|
||||
toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
|
||||
targets: ${{ inputs.targets }}
|
||||
components: ${{ inputs.components }}
|
||||
- name: Cache dependencies
|
||||
uses: actions/cache@v3
|
||||
if: inputs.cache-name != ''
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: ${{ runner.os }}-cargo-${{ inputs.cache-name }}-${{ steps.toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock') }}
|
||||
107
.github/workflows/ci.yml
vendored
107
.github/workflows/ci.yml
vendored
@ -1,107 +0,0 @@
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- mempool
|
||||
|
||||
|
||||
name: Compile Check and Lint
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Compile Check
|
||||
runs-on: mempool-ci
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup Rust
|
||||
uses: './.github/actions/ci-rust-setup'
|
||||
with:
|
||||
cache-name: dev
|
||||
- run: cargo check --all-features
|
||||
|
||||
fmt:
|
||||
name: Formatter
|
||||
runs-on: mempool-ci
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup Rust
|
||||
uses: './.github/actions/ci-rust-setup'
|
||||
with:
|
||||
components: rustfmt
|
||||
- run: cargo fmt --all -- --check
|
||||
|
||||
test:
|
||||
name: Run Tests
|
||||
runs-on: mempool-ci
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup Rust
|
||||
uses: './.github/actions/ci-rust-setup'
|
||||
with:
|
||||
cache-name: test
|
||||
- run: cargo test --lib --all-features
|
||||
|
||||
compile-freebsd:
|
||||
runs-on: mempool-ci
|
||||
name: Run Compile Checks in FreeBSD
|
||||
env:
|
||||
FREEBSD_VER: "14.3"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Cache dependencies for FreeBSD
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
.cargohome/registry
|
||||
.cargohome/git
|
||||
target
|
||||
key: freebsd-${{ env.FREEBSD_VER }}-cargo-checks-${{ hashFiles('**/Cargo.lock') }}
|
||||
- name: Compile Checks in FreeBSD
|
||||
uses: vmactions/freebsd-vm@v1
|
||||
with:
|
||||
usesh: true
|
||||
release: "${{ env.FREEBSD_VER }}"
|
||||
arch: amd64
|
||||
prepare: |
|
||||
mkdir -p ~/.cargo/
|
||||
mkdir -p ./.cargohome/registry/
|
||||
mkdir -p ./.cargohome/git/
|
||||
mv ./.cargohome/registry ~/.cargo/
|
||||
mv ./.cargohome/git ~/.cargo/
|
||||
rm -rf ./.cargohome
|
||||
pkg install -y git rsync gmake llvm rust rocksdb cmake
|
||||
|
||||
run: |
|
||||
cargo check --no-default-features
|
||||
cargo check -F liquid
|
||||
cargo check -F electrum-discovery
|
||||
cargo check -F electrum-discovery,liquid
|
||||
cargo build --release --bin electrs
|
||||
rm -rf ./.cargohome
|
||||
mkdir -p ~/.cargo/registry/
|
||||
mkdir -p ~/.cargo/git/
|
||||
mkdir -p ./.cargohome/
|
||||
mv ~/.cargo/registry ./.cargohome/
|
||||
mv ~/.cargo/git ./.cargohome/
|
||||
|
||||
clippy:
|
||||
name: Linter
|
||||
runs-on: mempool-ci
|
||||
needs: [check]
|
||||
strategy:
|
||||
matrix: # Try all combinations of features. Some times weird things appear.
|
||||
features: [
|
||||
'',
|
||||
'-F electrum-discovery',
|
||||
'-F liquid',
|
||||
'-F electrum-discovery,liquid',
|
||||
]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup Rust
|
||||
uses: './.github/actions/ci-rust-setup'
|
||||
with:
|
||||
cache-name: dev
|
||||
components: clippy
|
||||
- name: Clippy with Features = ${{ matrix.features }}
|
||||
run: cargo clippy ${{ matrix.features }} -- -D warnings
|
||||
77
.github/workflows/on-tag.yml
vendored
77
.github/workflows/on-tag.yml
vendored
@ -1,77 +0,0 @@
|
||||
name: Docker build on tag
|
||||
env:
|
||||
DOCKER_CLI_EXPERIMENTAL: enabled
|
||||
TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
|
||||
DOCKER_BUILDKIT: 0
|
||||
COMPOSE_DOCKER_CLI_BUILD: 0
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v[0-9]+.[0-9]+.[0-9]+
|
||||
- v[0-9]+.[0-9]+.[0-9]+-*
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: mempool-ci
|
||||
timeout-minutes: 120
|
||||
name: Build and push to DockerHub
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
include:
|
||||
- image: electrs
|
||||
cargo_extra_args: ""
|
||||
- image: electrs-liquid
|
||||
cargo_extra_args: "--features liquid"
|
||||
steps:
|
||||
- name: Set env variables
|
||||
run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
|
||||
|
||||
- name: Show set environment variables
|
||||
run: |
|
||||
printf " TAG: %s\n" "$TAG"
|
||||
|
||||
- name: Add SHORT_SHA env property with commit short sha
|
||||
run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
|
||||
|
||||
- name: Login to Docker for building
|
||||
run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
|
||||
|
||||
- name: Checkout project
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
id: qemu
|
||||
|
||||
- name: Setup Docker buildx action
|
||||
uses: docker/setup-buildx-action@v3
|
||||
id: buildx
|
||||
|
||||
- name: Available platforms
|
||||
run: echo ${{ steps.buildx.outputs.platforms }}
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v3
|
||||
id: cache
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx
|
||||
|
||||
- name: Run Docker buildx against tag
|
||||
run: |
|
||||
docker buildx build \
|
||||
--cache-from "type=local,src=/tmp/.buildx-cache" \
|
||||
--cache-to "type=local,dest=/tmp/.buildx-cache" \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.image }}:$TAG \
|
||||
--tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.image }}:latest \
|
||||
--output "type=registry" . \
|
||||
--build-arg commitHash=$SHORT_SHA \
|
||||
--build-arg CARGO_EXTRA_ARGS="${{ matrix.cargo_extra_args }}"
|
||||
18
.github/workflows/project-review-status.yml
vendored
18
.github/workflows/project-review-status.yml
vendored
@ -1,18 +0,0 @@
|
||||
name: Project Board Automation
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [review_requested]
|
||||
issues:
|
||||
types: [opened]
|
||||
|
||||
jobs:
|
||||
project-automation:
|
||||
uses: mempool/.github/.github/workflows/project-board-automation.yml@master
|
||||
with:
|
||||
project-number: 8
|
||||
secrets:
|
||||
PROJECT_TOKEN: ${{ secrets.PROJECT_TOKEN }}
|
||||
PROJECT_ID: ${{ secrets.PROJECT_ID }}
|
||||
STATUS_FIELD_ID: ${{ secrets.STATUS_FIELD_ID }}
|
||||
REVIEW_NEEDED_OPTION_ID: ${{ secrets.REVIEW_NEEDED_OPTION_ID }}
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@ -4,5 +4,3 @@ target
|
||||
*.sublime*
|
||||
*~
|
||||
*.pyc
|
||||
.vscode
|
||||
*.core
|
||||
|
||||
19
AGENTS.md
19
AGENTS.md
@ -1,19 +0,0 @@
|
||||
# electrs
|
||||
|
||||
## Rules
|
||||
|
||||
1. You are an expert Rust developer.
|
||||
2. You are an expert Bitcoin developer.
|
||||
3. If you are unsure of a change, ask the developer to make a choice proactively.
|
||||
|
||||
## Before testing
|
||||
|
||||
- Run cargo fmt (from root)
|
||||
- command: `cargo fmt`
|
||||
|
||||
## Testing
|
||||
|
||||
- Run the checks script
|
||||
- `./scripts/checks.sh`
|
||||
- Run with tests only when a test is added or changed
|
||||
- `INCLUDE_TESTS=1 ./scripts/checks.sh`
|
||||
2525
Cargo.lock
generated
2525
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
105
Cargo.toml
105
Cargo.toml
@ -1,69 +1,68 @@
|
||||
[package]
|
||||
name = "mempool-electrs"
|
||||
version = "3.4.0-dev"
|
||||
authors = [
|
||||
"Roman Zeyde <me@romanzey.de>",
|
||||
"Nadav Ivgi <nadav@shesek.info>",
|
||||
"wiz <j@wiz.biz>",
|
||||
"junderw <jonathan.underwood4649@gmail.com>"
|
||||
]
|
||||
name = "electrs"
|
||||
version = "0.4.1"
|
||||
authors = ["Roman Zeyde <me@romanzey.de>"]
|
||||
description = "An efficient re-implementation of Electrum Server in Rust"
|
||||
license = "MIT"
|
||||
homepage = "https://github.com/mempool/electrs"
|
||||
repository = "https://github.com/mempool/electrs"
|
||||
publish = false
|
||||
homepage = "https://github.com/romanz/electrs"
|
||||
repository = "https://github.com/romanz/electrs"
|
||||
keywords = ["bitcoin", "electrum", "server", "index", "database"]
|
||||
documentation = "https://docs.rs/electrs/"
|
||||
readme = "README.md"
|
||||
edition = "2018"
|
||||
|
||||
[lib]
|
||||
name = "electrs"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
liquid = ["elements"]
|
||||
electrum-discovery = ["electrum-client"]
|
||||
|
||||
[dependencies]
|
||||
arrayref = "0.3.6"
|
||||
base64 = "0.13.0"
|
||||
bincode-do-not-use-directly = { version = "1.3.1", package = "bincode" }
|
||||
bitcoin = { version = "0.32.8", features = [ "serde" ] }
|
||||
bounded-vec-deque = "0.1.1"
|
||||
clap = "2.33.3"
|
||||
crossbeam-channel = "0.5.0"
|
||||
dirs = "4.0.0"
|
||||
elements = { version = "0.26.1", features = [ "serde" ], optional = true }
|
||||
error-chain = "0.12.4"
|
||||
arraydeque = "0.4"
|
||||
arrayref = "0.3"
|
||||
base64 = "0.10"
|
||||
bincode = "1.0"
|
||||
bitcoin-bech32 = "0.9.0"
|
||||
chan = "0.1"
|
||||
chan-signal = "0.3"
|
||||
clap = "2.31"
|
||||
dirs = "1.0"
|
||||
error-chain = "0.12"
|
||||
futures = "0.1"
|
||||
glob = "0.3"
|
||||
hex = "0.4.2"
|
||||
itertools = "0.10"
|
||||
lazy_static = "1.3.0"
|
||||
hex = "0.3"
|
||||
hyper = "0.12.27"
|
||||
itertools = "0.8.0"
|
||||
libc = "0.2"
|
||||
log = "0.4.11"
|
||||
socket2 = { version = "0.4", features = ["all"] }
|
||||
num_cpus = "1.12.0"
|
||||
page_size = "0.4.2"
|
||||
prometheus = "0.13"
|
||||
ppp = "2.3.0"
|
||||
rayon = "1.5.0"
|
||||
rocksdb = "0.24.0"
|
||||
serde = "1.0.118"
|
||||
serde_derive = "1.0.118"
|
||||
serde_json = "1.0.60"
|
||||
sha2 = "0.10.7"
|
||||
signal-hook = "0.3"
|
||||
stderrlog = "0.5.0"
|
||||
time = { version = "0.3", features = ["formatting"] }
|
||||
tiny_http = "0.11"
|
||||
url = "2.2.0"
|
||||
hyper = "0.14"
|
||||
hyperlocal = "0.8"
|
||||
# close to same tokio version as dependent by hyper v0.14 and hyperlocal 0.8 -- things can go awry if they mismatch
|
||||
tokio = { version = "1", features = ["sync", "macros"] }
|
||||
log = "0.4"
|
||||
lru = "0.1"
|
||||
lru-cache = "0.1.1"
|
||||
num_cpus = "1.0"
|
||||
page_size = "0.4"
|
||||
prometheus = "0.5"
|
||||
rayon = "1.0"
|
||||
rocksdb = "0.12.1"
|
||||
rust-crypto = "0.2"
|
||||
secp256k1 = "0.12"
|
||||
serde = "1.0"
|
||||
serde_derive = "1.0"
|
||||
serde_json = "1.0"
|
||||
stderrlog = "0.4.1"
|
||||
sysconf = ">=0.3.4"
|
||||
time = "0.1"
|
||||
tiny_http = "0.6"
|
||||
url = "1.0"
|
||||
|
||||
# optional dependencies for electrum-discovery
|
||||
electrum-client = { version = "0.24.1", optional = true }
|
||||
[dependencies.bitcoin]
|
||||
version = "0.18"
|
||||
features = ["serde"]
|
||||
|
||||
[dependencies.bitcoin_hashes]
|
||||
version = "0.3"
|
||||
features = ["serde"]
|
||||
|
||||
[dependencies.elements]
|
||||
optional = true
|
||||
version = "0.7.0"
|
||||
features = ["serde-feature"] # Doesn't look to work by now
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.0"
|
||||
@ -72,7 +71,3 @@ tempfile = "3.0"
|
||||
lto = true
|
||||
panic = 'abort'
|
||||
codegen-units = 1
|
||||
|
||||
[patch.crates-io.electrum-client]
|
||||
git = "https://github.com/mempool/rust-electrum-client"
|
||||
rev = "4bbfc612d594fe23282c439d4bdc446cff01ba1c" # 0.24.1/add-peer branch
|
||||
|
||||
37
Dockerfile
37
Dockerfile
@ -1,30 +1,21 @@
|
||||
FROM debian:bookworm-slim AS base
|
||||
FROM rust:latest
|
||||
|
||||
ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y clang cmake
|
||||
|
||||
RUN apt update -qy && \
|
||||
apt install -qy librocksdb-dev curl
|
||||
RUN cargo install electrs
|
||||
|
||||
FROM base as build
|
||||
RUN adduser --disabled-login --system --shell /bin/false --uid 1000 user
|
||||
|
||||
RUN apt install -qy git clang cmake
|
||||
|
||||
ENV RUSTUP_HOME=/rust
|
||||
ENV CARGO_HOME=/cargo
|
||||
ENV PATH=/cargo/bin:/rust/bin:$PATH
|
||||
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
WORKDIR /build
|
||||
COPY . .
|
||||
|
||||
ARG CARGO_EXTRA_ARGS=""
|
||||
RUN cargo build --release --bin electrs ${CARGO_EXTRA_ARGS}
|
||||
|
||||
FROM base as deploy
|
||||
|
||||
COPY --from=build /build/target/release/electrs /bin/electrs
|
||||
USER user
|
||||
WORKDIR /home/user
|
||||
|
||||
# Electrum RPC
|
||||
EXPOSE 50001
|
||||
|
||||
ENTRYPOINT ["/bin/electrs"]
|
||||
# Prometheus monitoring
|
||||
EXPOSE 4224
|
||||
|
||||
STOPSIGNAL SIGINT
|
||||
|
||||
CMD ["electrs", "-vvvv", "--timestamp"]
|
||||
|
||||
61
README.md
61
README.md
@ -1,54 +1,54 @@
|
||||
# Mempool - Electrs backend API
|
||||
# Esplora - Electrs backend API
|
||||
|
||||
A block chain index engine and HTTP API written in Rust based on [romanz/electrs](https://github.com/romanz/electrs) and [Blockstream/electrs](https://github.com/Blockstream/electrs).
|
||||
A block chain index engine and HTTP API written in Rust based on [romanz/electrs](https://github.com/romanz/electrs).
|
||||
|
||||
Used as the backend for the [mempool block explorer](https://github.com/mempool/mempool) powering [mempool.space](https://mempool.space/).
|
||||
Used as the backend for the [Esplora block explorer](https://github.com/Blockstream/esplora) powering [blockstream.info](https://blockstream.info/).
|
||||
|
||||
API documentation [is available here](https://mempool.space/docs/api/rest).
|
||||
API documentation [is available here](https://github.com/blockstream/esplora/blob/master/API.md).
|
||||
|
||||
Documentation for the database schema and indexing process [is available here](doc/schema.md).
|
||||
|
||||
### Installing & indexing
|
||||
|
||||
Install Rust, Bitcoin Core (no `txindex` needed) and the `clang` and `cmake` packages, then:
|
||||
Install Rust, Bitcoin Core and the `clang` and `cmake` packages, then:
|
||||
|
||||
```bash
|
||||
$ git clone https://github.com/mempool/electrs && cd electrs
|
||||
$ git checkout mempool
|
||||
$ git clone https://github.com/blockstream/electrs && cd electrs
|
||||
$ git checkout new-index
|
||||
$ cargo run --release --bin electrs -- -vvvv --daemon-dir ~/.bitcoin
|
||||
|
||||
# Or for liquid:
|
||||
$ cargo run --features liquid --release --bin electrs -- -vvvv --network liquid --daemon-dir ~/.liquid
|
||||
$ cargo run --features liquid --release --bin electrs -- -vvvv --daemon-dir ~/.liquid
|
||||
```
|
||||
|
||||
See [electrs's original documentation](https://github.com/romanz/electrs/blob/master/doc/usage.md) for more detailed instructions.
|
||||
Note that our indexes are incompatible with electrs's and has to be created separately.
|
||||
|
||||
The indexes require 1.3TB of storage after running compaction (as of October 2023), but you'll need to have
|
||||
free space of about double that available during the index compaction process.
|
||||
Creating the indexes should take a few hours on a beefy machine with high speed NVMe SSD(s).
|
||||
The indexes require 440GB of storage after running compaction, but you'll need to have about 1TB
|
||||
of free space available for the initial (non-compacted) indexing process.
|
||||
Creating the indexes should take a few hours on a beefy machine with SSD.
|
||||
|
||||
### Light mode
|
||||
> Note: the pre-v2 version supported a "light" indexing mode for personal use,
|
||||
> where less data is kept on-disk in exchange for a performance hit and more reliance/load on bitcoind.
|
||||
> This option is not currently available, but can be enabled with [an older esplora release](https://github.com/Blockstream/esplora/releases/tag/esplora_v1.67).
|
||||
> We're hoping to eventually get this feature back. Let us know if you find this important!
|
||||
|
||||
For personal or low-volume use, you may set `--lightmode` to reduce disk storage requirements
|
||||
by roughly 50% at the cost of slower and more expensive lookups.
|
||||
|
||||
With this option set, raw transactions and metadata associated with blocks will not be kept in rocksdb
|
||||
(the `T`, `X` and `M` indexes),
|
||||
but instead queried from bitcoind on demand.
|
||||
To deploy with Docker, follow the [instructions here](https://github.com/Blockstream/esplora#how-to-build-the-docker-image).
|
||||
|
||||
### Notable changes from Electrs:
|
||||
|
||||
- HTTP REST API in addition to the Electrum JSON-RPC protocol, with extended transaction information
|
||||
- HTTP REST API instead of the Electrum JSON-RPC protocol, with extended transaction information
|
||||
(previous outputs, spending transactions, script asm and more).
|
||||
|
||||
- Extended indexes and database storage for improved performance under high load:
|
||||
|
||||
- A full transaction store mapping txids to raw transactions is kept in the database under the prefix `t`.
|
||||
- An index of all spendable transaction outputs is kept under the prefix `O`.
|
||||
- An index of all addresses (encoded as string) is kept under the prefix `a` to enable by-prefix address search.
|
||||
This takes up ~200GB of extra storage.
|
||||
- A map of blockhash to txids is kept in the database under the prefix `X`.
|
||||
- Block stats metadata (number of transactions, size and weight) is kept in the database under the prefix `M`.
|
||||
- The index with `T` prefix mapping txids to block heights now also includes the block hash.
|
||||
This allows for quick reorg-aware transaction confirmation status lookups, by verifying the
|
||||
current block at the recorded height still matches the recorded block hash.
|
||||
|
||||
With these new indexes, bitcoind is no longer queried to serve user requests and is only polled
|
||||
periodically for new blocks and for syncing the mempool.
|
||||
@ -61,22 +61,13 @@ but instead queried from bitcoind on demand.
|
||||
In addition to electrs's original configuration options, a few new options are also available:
|
||||
|
||||
- `--http-addr <addr:port>` - HTTP server address/port to listen on (default: `127.0.0.1:3000`).
|
||||
- `--lightmode` - enable light mode (see above)
|
||||
- `--disable-prevout` - disable attaching previous output information to inputs.
|
||||
This significantly reduces the amount of transaction lookups (and IO/CPU/memory usage),
|
||||
at the cost of not knowing inputs amounts, their previous script/address, and the transaction fee.
|
||||
- `--parent-network <network>` - the parent network this chain is pegged to (Elements/Liquid only).
|
||||
- `--cors <origins>` - origins allowed to make cross-site request (optional, defaults to none).
|
||||
- `--address-search` - enables the by-prefix address search index.
|
||||
- `--index-unspendables` - enables indexing of provably unspendable outputs.
|
||||
- `--utxos-limit <num>` - maximum number of utxos to return per address.
|
||||
- `--electrum-txs-limit <num>` - maximum number of txs to return per address in the electrum server (does not apply for the http api).
|
||||
- `--electrum-banner <text>` - welcome banner text for electrum server.
|
||||
|
||||
Additional options with the `liquid` feature:
|
||||
- `--parent-network <network>` - the parent network this chain is pegged to.
|
||||
|
||||
Additional options with the `electrum-discovery` feature:
|
||||
- `--electrum-hosts <json>` - a json map of the public hosts where the electrum server is reachable, in the [`server.features` format](https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server.features).
|
||||
- `--electrum-announce` - announce the electrum server on the electrum p2p server discovery network.
|
||||
|
||||
See `$ cargo run --bin electrs -- --help` for the full list of options.
|
||||
See `$ cargo run --release --bin electrs -- --help` for the full list of options.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
57
build.rs
57
build.rs
@ -1,57 +0,0 @@
|
||||
use std::{path::Path, process::Command};
|
||||
|
||||
fn main() {
|
||||
// Specify re-run conditions
|
||||
|
||||
// 1. Rerun build script if we pass a new GIT_HASH
|
||||
println!("cargo:rerun-if-env-changed=GIT_HASH");
|
||||
|
||||
// 2. Only do git based reruns if git directory exists
|
||||
if Path::new(".git").exists() {
|
||||
// If we change the branch, rerun
|
||||
println!("cargo:rerun-if-changed=.git/HEAD");
|
||||
if let Ok(r) = std::fs::read_to_string(".git/HEAD") {
|
||||
if let Some(stripped) = r.strip_prefix("ref: ") {
|
||||
// If the HEAD is detached it will be a commit hash
|
||||
// so the HEAD changed directive above will pick it up,
|
||||
// otherwise it will point to a ref in the refs directory
|
||||
println!("cargo:rerun-if-changed=.git/{}", stripped);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Getting git hash
|
||||
|
||||
// Don't fetch git hash if it's already in the ENV
|
||||
let existing = std::env::var("GIT_HASH").unwrap_or_else(|_| String::new());
|
||||
if !existing.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Get git hash from git and don't do anything if the command fails
|
||||
if let Some(rev_parse) = cmd("git", &["rev-parse", "--short", "HEAD"]) {
|
||||
// Add (dirty) to the GIT_HASH if the git status isn't clean
|
||||
// This includes untracked files
|
||||
let dirty = cmd("git", &["status", "--short"]).expect("git command works");
|
||||
|
||||
// Ignore Dockerfile deletion as it is expected in Docker buildx builds
|
||||
let git_hash = if dirty.is_empty() || dirty.trim() == "D Dockerfile" {
|
||||
rev_parse
|
||||
} else {
|
||||
format!("{}(dirty)", rev_parse.trim())
|
||||
};
|
||||
|
||||
println!("cargo:rustc-env=GIT_HASH={}", git_hash.trim());
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function, Command is verbose...
|
||||
fn cmd(name: &str, args: &[&str]) -> Option<String> {
|
||||
Command::new(name).args(args).output().ok().and_then(|o| {
|
||||
if o.status.success() {
|
||||
String::from_utf8(o.stdout).ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
10692
contrib/popular-scripts.txt
Normal file
10692
contrib/popular-scripts.txt
Normal file
File diff suppressed because it is too large
Load Diff
@ -37,38 +37,18 @@ Each output results in the following new row:
|
||||
|
||||
* `"O{txid}{vout}" → "{scriptpubkey}{value}"`
|
||||
|
||||
When the indexer is synced up to the tip of the chain, the hash of the tip is saved as following:
|
||||
|
||||
* `"t" → "{blockhash}"`
|
||||
|
||||
### `history`
|
||||
|
||||
Each funding output (except for provably unspendable ones when `--index-unspendables` is not enabled) results in the following new rows (`H` is for history, `F` is for funding):
|
||||
Each funding output (except for provably unspendable ones) results in the following new row (`H` is for history, `F` is for funding):
|
||||
|
||||
* `"H{funding-scripthash}{funding-height}F{funding-txid:vout}{value}" → ""`
|
||||
* `"a{funding-address-str}" → ""` (for prefix address search, only saved when `--address-search` is enabled)
|
||||
* `"H{funding-scripthash}{funding-height}F{funding-txid:index}{value}" → ""`
|
||||
|
||||
Each spending input (except the coinbase) results in the following new rows (`S` is for spending):
|
||||
|
||||
* `"H{funding-scripthash}{spending-height}S{spending-txid:vin}{funding-txid:vout}{value}" → ""`
|
||||
* `"H{funding-scripthash}{spending-height}S{spending-txid:index}{funding-txid:index}{value}" → ""`
|
||||
|
||||
* `"S{funding-txid:vout}{spending-txid:vin}" → ""`
|
||||
* `"S{funding-txid:index}{spending-txid:index}" → ""`
|
||||
|
||||
#### Elements only
|
||||
|
||||
Assets (re)issuances results in the following new rows (only for user-issued assets):
|
||||
|
||||
* `"i{asset-id}" → "{issuing-txid:vin}{prev-txid:vout}{issuance}{reissuance_token}"`
|
||||
* `"I{asset-id}{issuance-height}I{issuing-txid:vin}{is_reissuance}{amount}{tokens}" → ""`
|
||||
|
||||
Peg-ins/peg-outs results in the following new rows (only for the native asset, typically L-BTC):
|
||||
|
||||
* `"I{asset-id}{pegin-height}F{pegin-txid:vin}{value}" → ""`
|
||||
* `"I{asset-id}{pegout-height}F{pegout-txid:vout}{value}" → ""`
|
||||
|
||||
Every burn (unspendable output) results in the following new row (both user-issued and native):
|
||||
|
||||
* `"I{asset-id}{burn-height}F{burning-txid:vout}{value}" → ""`
|
||||
|
||||
### `cache`
|
||||
|
||||
@ -83,11 +63,3 @@ If the `blockhash` was since orphaned, the cache is removed and re-computed.
|
||||
* `"A{scripthash}" → "{stats}{blockhash}"` (where `stats` is composed of `tx_count`, `funded_txo_{count,sum}` and `spent_txo_{count,sum}`)
|
||||
|
||||
* `"U{scripthash}" → "{utxo}{blockhash}"` (where `utxo` is a set of `(txid,vout)` outpoints)
|
||||
|
||||
#### Elements only:
|
||||
|
||||
Stats for issued assets:
|
||||
* `"z{asset-id}" → "{issued_stats}{blockhash}"` (where `issued_stats` is composed of `tx_count`, `issuance_count`, `issued_amount`, `burned_amount`, `has_blinded_issuances`, `reissuance_tokens`, `burned_reissuance_tokens`)
|
||||
|
||||
Stats for the native asset:
|
||||
* `"z{issued-asset}" → "{native_stats}{blockhash}"` (where `native_stats` is composed of `tx_count`, `peg_in_count`, `peg_in_amount`, `peg_out_count`, `peg_out_amount`, `burn_count` and `burn_amount`)
|
||||
|
||||
@ -1,24 +0,0 @@
|
||||
version: '3.9'
|
||||
services:
|
||||
|
||||
mempool-electrs:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
restart: on-failure
|
||||
|
||||
ports:
|
||||
- 50001:50001
|
||||
|
||||
entrypoint:
|
||||
/bin/electrs
|
||||
command: |
|
||||
--address-search
|
||||
--cookie mempool:mempool
|
||||
--db-dir /electrs
|
||||
--cors '*'
|
||||
volumes:
|
||||
- 'electrs_data:/electrs'
|
||||
|
||||
volumes:
|
||||
electrs_data:
|
||||
@ -1,24 +0,0 @@
|
||||
#!/usr/bin/env zsh
|
||||
export ZPOOL=nvm
|
||||
|
||||
export BITCOIN_HOME=/bitcoin
|
||||
export BITCOIN_USER=bitcoin
|
||||
export BITCOIN_GROUP=bitcoin
|
||||
|
||||
export ELEMENTS_HOME=/elements
|
||||
export ELEMENTS_USER=elements
|
||||
export ELEMENTS_GROUP=elements
|
||||
|
||||
# create /bitcoin/socket with custom ACL for electrs unix sockets
|
||||
zfs create -o "mountpoint=${BITCOIN_HOME}/socket" "${ZPOOL}/bitcoin/socket"
|
||||
|
||||
# create /elements/socket with custom ACL for electrs unix sockets
|
||||
zfs create -o "mountpoint=${ELEMENTS_HOME}/socket" "${ZPOOL}/elements/socket"
|
||||
|
||||
setfacl -m "user:bitcoin:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${BITCOIN_HOME}/socket"
|
||||
|
||||
chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket"
|
||||
|
||||
setfacl -m "user:elements:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${ELEMENTS_HOME}/socket"
|
||||
|
||||
chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket"
|
||||
@ -1 +0,0 @@
|
||||
1.87
|
||||
@ -1,64 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# This script is used for running all the checks
|
||||
# needed to make sure CI passes.
|
||||
|
||||
# See below for the reasoning behind testing
|
||||
# clippy for all feature combinations.
|
||||
# (Note: clippy reports compile errors too)
|
||||
|
||||
# You can pass extra args to cargo commands
|
||||
# ./scripts/check.sh --release
|
||||
# will run in release mode. You can use that
|
||||
# if you already have release mode compiled, as
|
||||
# it will be faster to use the version you already compiled
|
||||
|
||||
TESTNAME=""
|
||||
cleanup() {
|
||||
exit_code=$?
|
||||
if [[ ${exit_code} -ne 0 ]]; then
|
||||
echo -e "\n\n##### Failed on \"$TESTNAME\"" 1>&2
|
||||
fi
|
||||
exit $exit_code
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
TESTNAME="Running cargo check"
|
||||
echo "$TESTNAME"
|
||||
cargo check $@ -q --all-features
|
||||
|
||||
TESTNAME="Running cargo fmt check"
|
||||
echo "$TESTNAME"
|
||||
cargo fmt $@ -q --all -- --check
|
||||
|
||||
# Testing all the combinations of clippy.
|
||||
# There were many instances where a certain struct
|
||||
# differed based on liquid or not(liquid) etc.
|
||||
# and the clippy fixes would break the other
|
||||
# feature combination.
|
||||
#
|
||||
# Be prepared to use #[allow(clippy::___)] attributes
|
||||
# to "fix" contradictions between feature sets.
|
||||
|
||||
TESTNAME="Running cargo clippy check no features"
|
||||
echo "$TESTNAME"
|
||||
cargo clippy $@ -q
|
||||
|
||||
TESTNAME="Running cargo clippy check electrum-discovery"
|
||||
echo "$TESTNAME"
|
||||
cargo clippy $@ -q -F electrum-discovery
|
||||
|
||||
TESTNAME="Running cargo clippy check liquid"
|
||||
echo "$TESTNAME"
|
||||
cargo clippy $@ -q -F liquid
|
||||
|
||||
TESTNAME="Running cargo clippy check electrum-discovery + liquid"
|
||||
echo "$TESTNAME"
|
||||
cargo clippy $@ -q -F electrum-discovery,liquid
|
||||
|
||||
if [ $INCLUDE_TESTS ]; then
|
||||
TESTNAME="Running cargo test with all features"
|
||||
echo "$TESTNAME"
|
||||
cargo test $@ -q --lib --all-features
|
||||
fi
|
||||
@ -1,4 +1,4 @@
|
||||
use bitcoin::hashes::sha256d::Hash as Sha256dHash;
|
||||
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::{daemon, index, signal::Waiter, store};
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
extern crate bitcoin;
|
||||
extern crate error_chain;
|
||||
#[macro_use]
|
||||
extern crate log;
|
||||
@ -20,123 +21,75 @@ use electrs::{
|
||||
signal::Waiter,
|
||||
};
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
use electrs::elements::AssetRegistry;
|
||||
|
||||
fn fetch_from(config: &Config, store: &Store) -> FetchFrom {
|
||||
let mut jsonrpc_import = config.jsonrpc_import;
|
||||
if !jsonrpc_import {
|
||||
// switch over to jsonrpc after the initial sync is done
|
||||
jsonrpc_import = store.done_initial_sync();
|
||||
jsonrpc_import = !store.is_empty();
|
||||
}
|
||||
match jsonrpc_import {
|
||||
true => FetchFrom::Bitcoind, // slower, uses JSONRPC (good for incremental updates)
|
||||
false => FetchFrom::BlkFiles, // faster, uses blk*.dat files (good for initial indexing)
|
||||
}
|
||||
}
|
||||
|
||||
if jsonrpc_import {
|
||||
// slower, uses JSONRPC (good for incremental updates)
|
||||
FetchFrom::Bitcoind
|
||||
} else {
|
||||
// faster, uses blk*.dat files (good for initial indexing)
|
||||
FetchFrom::BlkFiles
|
||||
fn finish_verification(daemon: &Daemon, signal: &Waiter) -> Result<()> {
|
||||
loop {
|
||||
let progress = daemon.getblockchaininfo()?.verificationprogress;
|
||||
if progress > 0.9999 {
|
||||
return Ok(());
|
||||
}
|
||||
warn!(
|
||||
"waiting for verification to finish: {:.3}%",
|
||||
progress * 100.0
|
||||
);
|
||||
signal.wait(Duration::from_secs(5))?;
|
||||
}
|
||||
}
|
||||
|
||||
fn run_server(config: Arc<Config>) -> Result<()> {
|
||||
let signal = Waiter::start();
|
||||
let signal = Waiter::new();
|
||||
let metrics = Metrics::new(config.monitoring_addr);
|
||||
metrics.start();
|
||||
|
||||
let daemon = Arc::new(Daemon::new(
|
||||
config.daemon_dir.clone(),
|
||||
config.blocks_dir.clone(),
|
||||
&config.daemon_dir,
|
||||
config.daemon_rpc_addr,
|
||||
config.cookie_getter(),
|
||||
config.network_type,
|
||||
config.magic,
|
||||
signal.clone(),
|
||||
&metrics,
|
||||
)?);
|
||||
let store = Arc::new(Store::open(&config.db_path.join("newindex"), &config));
|
||||
let mut indexer = Indexer::open(
|
||||
Arc::clone(&store),
|
||||
fetch_from(&config, &store),
|
||||
&config,
|
||||
&metrics,
|
||||
);
|
||||
finish_verification(&daemon, &signal)?;
|
||||
let store = Arc::new(Store::open(&config.db_path.join("newindex")));
|
||||
let mut indexer = Indexer::open(Arc::clone(&store), fetch_from(&config, &store), &metrics);
|
||||
let mut tip = indexer.update(&daemon)?;
|
||||
|
||||
let chain = Arc::new(ChainQuery::new(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&daemon),
|
||||
&config,
|
||||
&metrics,
|
||||
));
|
||||
let chain = Arc::new(ChainQuery::new(Arc::clone(&store), &metrics));
|
||||
|
||||
let mempool = Arc::new(RwLock::new(Mempool::new(
|
||||
Arc::clone(&chain),
|
||||
&metrics,
|
||||
Arc::clone(&config),
|
||||
)));
|
||||
loop {
|
||||
match Mempool::update(&mempool, &daemon) {
|
||||
Ok(_) => break,
|
||||
Err(e) => {
|
||||
warn!(
|
||||
"Error performing initial mempool update, trying again in 5 seconds: {}",
|
||||
e.display_chain()
|
||||
);
|
||||
signal.wait(Duration::from_secs(5), false)?;
|
||||
}
|
||||
}
|
||||
if let Some(ref precache_file) = config.precache_scripts {
|
||||
let precache_scripthashes = precache::scripthashes_from_file(precache_file.to_string())
|
||||
.expect("cannot load scripts to precache");
|
||||
precache::precache(&chain, precache_scripthashes);
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
let asset_db = config.asset_db_path.as_ref().map(|db_dir| {
|
||||
let asset_db = Arc::new(RwLock::new(AssetRegistry::new(db_dir.clone())));
|
||||
AssetRegistry::spawn_sync(asset_db.clone());
|
||||
asset_db
|
||||
});
|
||||
let mempool = Arc::new(RwLock::new(Mempool::new(Arc::clone(&chain), &metrics)));
|
||||
mempool.write().unwrap().update(&daemon)?;
|
||||
|
||||
let query = Arc::new(Query::new(
|
||||
Arc::clone(&chain),
|
||||
Arc::clone(&mempool),
|
||||
Arc::clone(&daemon),
|
||||
Arc::clone(&config),
|
||||
#[cfg(feature = "liquid")]
|
||||
asset_db,
|
||||
));
|
||||
|
||||
// TODO: configuration for which servers to start
|
||||
let rest_server = rest::start(Arc::clone(&config), Arc::clone(&query), &metrics);
|
||||
let electrum_server = ElectrumRPC::start(Arc::clone(&config), Arc::clone(&query), &metrics);
|
||||
|
||||
if let Some(ref precache_file) = config.precache_scripts {
|
||||
let precache_scripthashes = precache::scripthashes_from_file(precache_file.to_string())
|
||||
.expect("cannot load scripts to precache");
|
||||
precache::precache(
|
||||
Arc::clone(&chain),
|
||||
precache_scripthashes,
|
||||
config.precache_threads,
|
||||
);
|
||||
}
|
||||
let rest_server = rest::run_server(Arc::clone(&config), Arc::clone(&query));
|
||||
let electrum_server =
|
||||
ElectrumRPC::start(config.electrum_rpc_addr, Arc::clone(&query), &metrics);
|
||||
|
||||
loop {
|
||||
if let Err(err) = signal.wait(Duration::from_millis(config.main_loop_delay), true) {
|
||||
if let Err(err) = signal.wait(Duration::from_secs(5)) {
|
||||
info!("stopping server: {}", err);
|
||||
|
||||
electrs::util::spawn_thread("shutdown-thread-checker", || {
|
||||
let mut counter = 40;
|
||||
let interval_ms = 500;
|
||||
|
||||
while counter > 0 {
|
||||
electrs::util::with_spawned_threads(|threads| {
|
||||
debug!("Threads during shutdown: {:?}", threads);
|
||||
});
|
||||
std::thread::sleep(std::time::Duration::from_millis(interval_ms));
|
||||
counter -= 1;
|
||||
}
|
||||
});
|
||||
|
||||
rest_server.stop();
|
||||
// the electrum server is stopped when dropped
|
||||
break;
|
||||
}
|
||||
|
||||
@ -148,13 +101,7 @@ fn run_server(config: Arc<Config>) -> Result<()> {
|
||||
};
|
||||
|
||||
// Update mempool
|
||||
if let Err(e) = Mempool::update(&mempool, &daemon) {
|
||||
// Log the error if the result is an Err
|
||||
warn!(
|
||||
"Error updating mempool, skipping mempool update: {}",
|
||||
e.display_chain()
|
||||
);
|
||||
}
|
||||
mempool.write().unwrap().update(&daemon)?;
|
||||
|
||||
// Update subscribed clients
|
||||
electrum_server.notify();
|
||||
@ -169,7 +116,4 @@ fn main() {
|
||||
error!("server failed: {}", e.display_chain());
|
||||
process::exit(1);
|
||||
}
|
||||
electrs::util::with_spawned_threads(|threads| {
|
||||
debug!("Threads before closing: {:?}", threads);
|
||||
});
|
||||
}
|
||||
|
||||
@ -1,183 +1,48 @@
|
||||
extern crate electrs;
|
||||
|
||||
use std::{convert::TryInto, thread::ThreadId, time::Instant};
|
||||
use bincode;
|
||||
use electrs::{
|
||||
config::Config,
|
||||
new_index::{Store, TxHistoryKey},
|
||||
};
|
||||
|
||||
use electrs::{config::Config, new_index::db::open_raw_db};
|
||||
use lazy_static::lazy_static;
|
||||
|
||||
/*
|
||||
// How to run:
|
||||
export ELECTRS_DATA=/path/to/electrs
|
||||
cargo run \
|
||||
-q --release --bin popular-scripts -- \
|
||||
--db-dir $ELECTRS_DATA/db \
|
||||
> ./contrib/popular-scripts.txt
|
||||
*/
|
||||
|
||||
type DB = rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>;
|
||||
lazy_static! {
|
||||
static ref HISTORY_DB: DB = {
|
||||
let config = Config::from_args();
|
||||
open_raw_db(
|
||||
&config.db_path.join("newindex").join("history"),
|
||||
electrs::new_index::db::OpenMode::ReadOnly,
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
// Dev note:
|
||||
// Only use println for file output (lines for output)
|
||||
// Use eprintln to print to stderr for dev notifications
|
||||
fn main() {
|
||||
let high_usage_threshold = std::env::var("HIGH_USAGE_THRESHOLD")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<u32>().ok())
|
||||
.unwrap_or(4000);
|
||||
let thread_count = std::env::var("JOB_THREAD_COUNT")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<usize>().ok())
|
||||
.unwrap_or(4);
|
||||
eprintln!(
|
||||
"Seaching for scripts with history rows of {} or more...",
|
||||
high_usage_threshold
|
||||
);
|
||||
let config = Config::from_args();
|
||||
let store = Store::open(&config.db_path.join("newindex"));
|
||||
|
||||
let thread_pool = rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(thread_count)
|
||||
.build()
|
||||
.expect("Built threadpool");
|
||||
|
||||
let (sender, receiver) = crossbeam_channel::unbounded::<[u8; 32]>();
|
||||
|
||||
let increment = 256 / thread_count;
|
||||
let bytes: Vec<u8> = (0u8..=255u8)
|
||||
.filter(|n| *n % increment as u8 == 0)
|
||||
.collect();
|
||||
|
||||
let now = Instant::now();
|
||||
for i in 0..bytes.len() {
|
||||
let sender = sender.clone();
|
||||
let first_byte = bytes[i];
|
||||
let second_byte = bytes.get(i + 1).copied();
|
||||
|
||||
thread_pool.spawn(move || {
|
||||
let id = std::thread::current().id();
|
||||
run_iterator(
|
||||
id,
|
||||
&HISTORY_DB,
|
||||
high_usage_threshold,
|
||||
first_byte,
|
||||
second_byte,
|
||||
sender,
|
||||
now,
|
||||
);
|
||||
eprintln!("{id:?} Finished its job!");
|
||||
})
|
||||
}
|
||||
// If we don't drop this sender
|
||||
// the receiver will hang forever
|
||||
drop(sender);
|
||||
|
||||
while let Ok(script) = receiver.recv() {
|
||||
println!("{}", hex::encode(script));
|
||||
}
|
||||
eprintln!("Finished!!!!");
|
||||
}
|
||||
|
||||
fn run_iterator(
|
||||
thread_id: ThreadId,
|
||||
db: &DB,
|
||||
high_usage_threshold: u32,
|
||||
first_byte: u8,
|
||||
next_byte: Option<u8>,
|
||||
sender: crossbeam_channel::Sender<[u8; 32]>,
|
||||
now: Instant,
|
||||
) {
|
||||
let mut iter = db.raw_iterator();
|
||||
eprintln!(
|
||||
"Thread ({thread_id:?}) Seeking DB to beginning of tx histories for b'H' + {}",
|
||||
hex::encode([first_byte])
|
||||
);
|
||||
let mut compare_vec: Vec<u8> = vec![b'H', first_byte];
|
||||
iter.seek(&compare_vec); // Seek to beginning of our section
|
||||
|
||||
// Insert the byte of the next section for comparing
|
||||
// This will tell us when to stop with a closure
|
||||
type Checker<'a> = Box<dyn Fn(&[u8]) -> bool + 'a>;
|
||||
let is_finished: Checker<'_> = if let Some(next) = next_byte {
|
||||
// Modify the vec to what we're looking for next
|
||||
// to indicate we left our section
|
||||
compare_vec[1] = next;
|
||||
Box::new(|key: &[u8]| -> bool { key.starts_with(&compare_vec) })
|
||||
} else {
|
||||
// Modify the vec to only have H so we know when we left H
|
||||
compare_vec.remove(1);
|
||||
Box::new(|key: &[u8]| -> bool { !key.starts_with(&compare_vec) })
|
||||
};
|
||||
|
||||
eprintln!("Thread ({thread_id:?}) Seeking done");
|
||||
let mut iter = store.history_db().raw_iterator();
|
||||
iter.seek(b"H");
|
||||
|
||||
let mut curr_scripthash = [0u8; 32];
|
||||
let mut total_entries: usize = 0;
|
||||
let mut iter_index: usize = 1;
|
||||
let mut total_entries = 0;
|
||||
|
||||
while iter.valid() {
|
||||
let key = iter.key().unwrap();
|
||||
|
||||
if key.is_empty() || key[0] != b'H' || is_finished(key) {
|
||||
// We have left the txhistory section,
|
||||
// but we need to check the final scripthash
|
||||
send_if_popular(
|
||||
high_usage_threshold,
|
||||
total_entries,
|
||||
curr_scripthash,
|
||||
&sender,
|
||||
);
|
||||
if !key.starts_with(b"H") {
|
||||
break;
|
||||
}
|
||||
|
||||
if iter_index % 10_000_000 == 0 {
|
||||
let duration = now.elapsed().as_secs();
|
||||
eprintln!(
|
||||
"Thread ({thread_id:?}) Processing row #{iter_index}... {duration} seconds elapsed"
|
||||
);
|
||||
}
|
||||
let entry: TxHistoryKey = bincode::config()
|
||||
.big_endian()
|
||||
.deserialize(&key)
|
||||
.expect("failed to deserialize TxHistoryKey");
|
||||
|
||||
// We know that the TxHistory key is 1 byte "H" followed by
|
||||
// 32 byte scripthash
|
||||
let entry_hash: [u8; 32] = key[1..33].try_into().unwrap();
|
||||
if curr_scripthash != entry.scripthash {
|
||||
if total_entries > 100 {
|
||||
println!("{} {}", hex::encode(&curr_scripthash), total_entries);
|
||||
}
|
||||
|
||||
if curr_scripthash != entry_hash {
|
||||
// We have rolled on to a new scripthash
|
||||
// If the last scripthash was popular
|
||||
// Collect for sorting
|
||||
send_if_popular(
|
||||
high_usage_threshold,
|
||||
total_entries,
|
||||
curr_scripthash,
|
||||
&sender,
|
||||
);
|
||||
|
||||
// After collecting, reset values for next scripthash
|
||||
curr_scripthash = entry_hash;
|
||||
curr_scripthash = entry.scripthash;
|
||||
total_entries = 0;
|
||||
}
|
||||
|
||||
total_entries += 1;
|
||||
iter_index += 1;
|
||||
total_entries = total_entries + 1;
|
||||
|
||||
iter.next();
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn send_if_popular(
|
||||
high_usage_threshold: u32,
|
||||
total_entries: usize,
|
||||
curr_scripthash: [u8; 32],
|
||||
sender: &crossbeam_channel::Sender<[u8; 32]>,
|
||||
) {
|
||||
if total_entries >= high_usage_threshold as usize {
|
||||
sender.send(curr_scripthash).unwrap();
|
||||
if total_entries > 100 {
|
||||
println!("{} {}", hex::encode(&curr_scripthash), total_entries);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,4 @@
|
||||
extern crate electrs;
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
#[macro_use]
|
||||
extern crate log;
|
||||
|
||||
@ -9,7 +7,7 @@ fn main() {
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use bitcoin::blockdata::script::ScriptBuf;
|
||||
use bitcoin::blockdata::script::Script;
|
||||
use bitcoin::consensus::encode::deserialize;
|
||||
use electrs::{
|
||||
chain::Transaction,
|
||||
@ -21,30 +19,28 @@ fn main() {
|
||||
util::has_prevout,
|
||||
};
|
||||
|
||||
let signal = Waiter::start();
|
||||
let signal = Waiter::new();
|
||||
let config = Config::from_args();
|
||||
let store = Arc::new(Store::open(&config.db_path.join("newindex"), &config));
|
||||
let store = Arc::new(Store::open(&config.db_path.join("newindex")));
|
||||
|
||||
let metrics = Metrics::new(config.monitoring_addr);
|
||||
metrics.start();
|
||||
|
||||
let chain = ChainQuery::new(Arc::clone(&store), &metrics);
|
||||
|
||||
let daemon = Arc::new(
|
||||
Daemon::new(
|
||||
config.daemon_dir.clone(),
|
||||
config.blocks_dir.clone(),
|
||||
&config.daemon_dir,
|
||||
config.daemon_rpc_addr,
|
||||
config.cookie_getter(),
|
||||
config.network_type,
|
||||
config.magic,
|
||||
signal,
|
||||
signal.clone(),
|
||||
&metrics,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
let chain = ChainQuery::new(Arc::clone(&store), Arc::clone(&daemon), &config, &metrics);
|
||||
|
||||
let mut indexer = Indexer::open(Arc::clone(&store), FetchFrom::Bitcoind, &config, &metrics);
|
||||
let mut indexer = Indexer::open(Arc::clone(&store), FetchFrom::Bitcoind, &metrics);
|
||||
indexer.update(&daemon).unwrap();
|
||||
|
||||
let mut iter = store.txstore_db().raw_iterator();
|
||||
@ -61,17 +57,17 @@ fn main() {
|
||||
break;
|
||||
}
|
||||
|
||||
let tx: Transaction = deserialize(value).expect("failed to parse Transaction");
|
||||
let txid = tx.compute_txid();
|
||||
|
||||
iter.next();
|
||||
|
||||
let tx: Transaction = deserialize(&value).expect("failed to parse Transaction");
|
||||
let txid = tx.txid();
|
||||
|
||||
// only consider transactions of exactly two outputs
|
||||
if tx.output.len() != 2 {
|
||||
continue;
|
||||
}
|
||||
// skip coinbase txs
|
||||
if tx.is_coinbase() {
|
||||
if tx.is_coin_base() {
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -91,26 +87,12 @@ fn main() {
|
||||
.collect(),
|
||||
);
|
||||
|
||||
let total_out: u64 = tx.output.iter().map(|out| out.value.to_sat()).sum();
|
||||
let small_out = tx
|
||||
.output
|
||||
.iter()
|
||||
.map(|out| out.value.to_sat())
|
||||
.min()
|
||||
.unwrap();
|
||||
let large_out = tx
|
||||
.output
|
||||
.iter()
|
||||
.map(|out| out.value.to_sat())
|
||||
.max()
|
||||
.unwrap();
|
||||
let total_out: u64 = tx.output.iter().map(|out| out.value).sum();
|
||||
let small_out = tx.output.iter().map(|out| out.value).min().unwrap();
|
||||
let large_out = tx.output.iter().map(|out| out.value).max().unwrap();
|
||||
|
||||
let total_in: u64 = prevouts.values().map(|out| out.value.to_sat()).sum();
|
||||
let smallest_in = prevouts
|
||||
.values()
|
||||
.map(|out| out.value.to_sat())
|
||||
.min()
|
||||
.unwrap();
|
||||
let total_in: u64 = prevouts.values().map(|out| out.value).sum();
|
||||
let smallest_in = prevouts.values().map(|out| out.value).min().unwrap();
|
||||
|
||||
let fee = total_in - total_out;
|
||||
|
||||
@ -133,7 +115,7 @@ fn main() {
|
||||
|
||||
// test for sending back to one of the spent spks
|
||||
let has_reuse = {
|
||||
let prev_spks: HashSet<ScriptBuf> = prevouts
|
||||
let prev_spks: HashSet<Script> = prevouts
|
||||
.values()
|
||||
.map(|out| out.script_pubkey.clone())
|
||||
.collect();
|
||||
@ -147,8 +129,8 @@ fn main() {
|
||||
txid, blockid.height, tx.lock_time, uih, is_multi_spend as u8, has_reuse as u8
|
||||
);
|
||||
|
||||
total += 1;
|
||||
uih_totals[uih] += 1;
|
||||
total = total + 1;
|
||||
uih_totals[uih] = uih_totals[uih] + 1;
|
||||
}
|
||||
info!(
|
||||
"processed {} total txs, UIH counts: {:?}",
|
||||
|
||||
263
src/chain.rs
263
src/chain.rs
@ -1,65 +1,14 @@
|
||||
use std::str::FromStr;
|
||||
|
||||
#[cfg(not(feature = "liquid"))] // use regular Bitcoin data structures
|
||||
pub use bitcoin::{
|
||||
address,
|
||||
block::Header as BlockHeader,
|
||||
blockdata::{opcodes, script},
|
||||
consensus::deserialize,
|
||||
hashes, Block, BlockHash, OutPoint, ScriptBuf as Script, Transaction, TxIn, TxOut, Txid,
|
||||
Witness,
|
||||
};
|
||||
pub use bitcoin::{Block, BlockHeader, OutPoint, Transaction, TxIn, TxOut};
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub use {
|
||||
crate::elements::asset,
|
||||
elements::{
|
||||
address, bitcoin::bech32::Hrp, confidential, encode::deserialize, hashes, opcodes, script,
|
||||
Address, AssetId, Block, BlockHash, BlockHeader, OutPoint, Script, Transaction, TxIn,
|
||||
TxInWitness as Witness, TxOut, Txid,
|
||||
},
|
||||
};
|
||||
pub use elements::{confidential, Block, BlockHeader, OutPoint, Transaction, TxIn, TxOut};
|
||||
|
||||
use bitcoin::blockdata::constants::genesis_block;
|
||||
pub use bitcoin::Network as BNetwork;
|
||||
|
||||
// Extension trait for getting txid in a cross-compatible way
|
||||
pub trait TxidCompat {
|
||||
fn get_txid(&self) -> Txid;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
impl TxidCompat for Transaction {
|
||||
fn get_txid(&self) -> Txid {
|
||||
self.compute_txid()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
impl TxidCompat for Transaction {
|
||||
fn get_txid(&self) -> Txid {
|
||||
self.txid()
|
||||
}
|
||||
}
|
||||
|
||||
// Extension trait for getting block size in a cross-compatible way
|
||||
pub trait BlockSizeCompat {
|
||||
fn get_block_size(&self) -> usize;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
impl BlockSizeCompat for Block {
|
||||
fn get_block_size(&self) -> usize {
|
||||
self.total_size()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
impl BlockSizeCompat for Block {
|
||||
fn get_block_size(&self) -> usize {
|
||||
self.size()
|
||||
}
|
||||
}
|
||||
use bitcoin::network::constants::Network as BNetwork;
|
||||
use bitcoin::util::hash::BitcoinHash;
|
||||
use bitcoin_bech32::constants::Network as B32Network;
|
||||
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
pub type Value = u64;
|
||||
@ -68,92 +17,32 @@ pub use confidential::Value;
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Hash, Serialize, Ord, PartialOrd, Eq)]
|
||||
pub enum Network {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Bitcoin,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Testnet,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Testnet4,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Regtest,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Signet,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
Liquid,
|
||||
#[cfg(feature = "liquid")]
|
||||
LiquidTestnet,
|
||||
#[cfg(feature = "liquid")]
|
||||
LiquidRegtest,
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub const LIQUID_TESTNET_PARAMS: address::AddressParams = address::AddressParams {
|
||||
p2pkh_prefix: 36,
|
||||
p2sh_prefix: 19,
|
||||
blinded_prefix: 23,
|
||||
bech_hrp: Hrp::parse_unchecked("tex"),
|
||||
blech_hrp: Hrp::parse_unchecked("tlq"),
|
||||
};
|
||||
|
||||
/// Magic for testnet4, 0x1c163f28 (from BIP94) with flipped endianness.
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
const TESTNET4_MAGIC: u32 = 0x283f161c;
|
||||
|
||||
impl Network {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
pub fn magic(self) -> u32 {
|
||||
match self {
|
||||
Self::Testnet4 => TESTNET4_MAGIC,
|
||||
_ => {
|
||||
let magic = BNetwork::from(self).magic();
|
||||
u32::from_le_bytes(magic.to_bytes())
|
||||
}
|
||||
}
|
||||
pub fn genesis_hash(&self) -> Sha256dHash {
|
||||
let block = genesis_block(BNetwork::from(self));
|
||||
block.bitcoin_hash()
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn magic(self) -> u32 {
|
||||
pub fn magic(&self) -> u32 {
|
||||
match self {
|
||||
Network::Liquid | Network::LiquidRegtest => 0xDAB5_BFFA,
|
||||
Network::LiquidTestnet => 0x62DD_0E41,
|
||||
}
|
||||
}
|
||||
Network::Bitcoin => 0xD9B4BEF9,
|
||||
Network::Testnet => 0x0709110B,
|
||||
Network::Regtest => 0xDAB5BFFA,
|
||||
|
||||
pub fn is_regtest(self) -> bool {
|
||||
match self {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Regtest => true,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidRegtest => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn address_params(self) -> &'static address::AddressParams {
|
||||
// Liquid regtest uses elements's address params
|
||||
match self {
|
||||
Network::Liquid => &address::AddressParams::LIQUID,
|
||||
Network::LiquidRegtest => &address::AddressParams::ELEMENTS,
|
||||
Network::LiquidTestnet => &LIQUID_TESTNET_PARAMS,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn native_asset(self) -> &'static AssetId {
|
||||
match self {
|
||||
Network::Liquid => &asset::NATIVE_ASSET_ID,
|
||||
Network::LiquidTestnet => &asset::NATIVE_ASSET_ID_TESTNET,
|
||||
Network::LiquidRegtest => &asset::NATIVE_ASSET_ID_REGTEST,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn pegged_asset(self) -> Option<&'static AssetId> {
|
||||
match self {
|
||||
Network::Liquid => Some(&*asset::NATIVE_ASSET_ID),
|
||||
Network::LiquidTestnet | Network::LiquidRegtest => None,
|
||||
Network::Liquid => 0xDAB5BFFA,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidRegtest => 0xDAB5BFFA,
|
||||
}
|
||||
}
|
||||
|
||||
@ -163,97 +52,29 @@ impl Network {
|
||||
"mainnet".to_string(),
|
||||
"testnet".to_string(),
|
||||
"regtest".to_string(),
|
||||
"signet".to_string(),
|
||||
];
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
return vec![
|
||||
"mainnet".to_string(),
|
||||
"testnet".to_string(),
|
||||
"regtest".to_string(),
|
||||
"liquid".to_string(),
|
||||
"liquidtestnet".to_string(),
|
||||
"liquidregtest".to_string(),
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
pub fn genesis_hash(network: Network) -> BlockHash {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
return bitcoin_genesis_hash(network);
|
||||
#[cfg(feature = "liquid")]
|
||||
return liquid_genesis_hash(network);
|
||||
}
|
||||
|
||||
pub fn bitcoin_genesis_hash(network: Network) -> bitcoin::BlockHash {
|
||||
lazy_static! {
|
||||
static ref BITCOIN_GENESIS: bitcoin::BlockHash =
|
||||
genesis_block(BNetwork::Bitcoin).block_hash();
|
||||
static ref TESTNET_GENESIS: bitcoin::BlockHash =
|
||||
genesis_block(BNetwork::Testnet).block_hash();
|
||||
static ref TESTNET4_GENESIS: bitcoin::BlockHash = bitcoin::BlockHash::from_str(
|
||||
"00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043"
|
||||
)
|
||||
.unwrap();
|
||||
static ref REGTEST_GENESIS: bitcoin::BlockHash =
|
||||
genesis_block(BNetwork::Regtest).block_hash();
|
||||
static ref SIGNET_GENESIS: bitcoin::BlockHash =
|
||||
genesis_block(BNetwork::Signet).block_hash();
|
||||
}
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
match network {
|
||||
Network::Bitcoin => *BITCOIN_GENESIS,
|
||||
Network::Testnet => *TESTNET_GENESIS,
|
||||
Network::Testnet4 => *TESTNET4_GENESIS,
|
||||
Network::Regtest => *REGTEST_GENESIS,
|
||||
Network::Signet => *SIGNET_GENESIS,
|
||||
}
|
||||
#[cfg(feature = "liquid")]
|
||||
match network {
|
||||
Network::Liquid => *BITCOIN_GENESIS,
|
||||
Network::LiquidTestnet => *TESTNET_GENESIS,
|
||||
Network::LiquidRegtest => *REGTEST_GENESIS,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn liquid_genesis_hash(network: Network) -> elements::BlockHash {
|
||||
lazy_static! {
|
||||
static ref LIQUID_GENESIS: BlockHash =
|
||||
"1466275836220db2944ca059a3a10ef6fd2ea684b0688d2c379296888a206003"
|
||||
.parse()
|
||||
.unwrap();
|
||||
static ref ZERO_HASH: BlockHash =
|
||||
"0000000000000000000000000000000000000000000000000000000000000000"
|
||||
.parse()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
match network {
|
||||
Network::Liquid => *LIQUID_GENESIS,
|
||||
// The genesis block for liquid regtest chains varies based on the chain configuration.
|
||||
// This instead uses an all zeroed-out hash, which doesn't matter in practice because its
|
||||
// only used for Electrum server discovery, which isn't active on regtest.
|
||||
_ => *ZERO_HASH,
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for Network {
|
||||
fn from(network_name: &str) -> Self {
|
||||
match network_name {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
"mainnet" => Network::Bitcoin,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
"testnet" => Network::Testnet,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
"testnet4" => Network::Testnet4,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
"regtest" => Network::Regtest,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
"signet" => Network::Signet,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
"liquid" => Network::Liquid,
|
||||
#[cfg(feature = "liquid")]
|
||||
"liquidtestnet" => Network::LiquidTestnet,
|
||||
#[cfg(feature = "liquid")]
|
||||
"liquidregtest" => Network::LiquidRegtest,
|
||||
|
||||
_ => panic!("unsupported Bitcoin network: {:?}", network_name),
|
||||
@ -261,28 +82,48 @@ impl From<&str> for Network {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
impl From<Network> for BNetwork {
|
||||
fn from(network: Network) -> Self {
|
||||
impl From<&Network> for BNetwork {
|
||||
fn from(network: &Network) -> Self {
|
||||
match network {
|
||||
Network::Bitcoin => BNetwork::Bitcoin,
|
||||
Network::Testnet => BNetwork::Testnet,
|
||||
Network::Testnet4 => BNetwork::Testnet4,
|
||||
Network::Regtest => BNetwork::Regtest,
|
||||
Network::Signet => BNetwork::Signet,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::Liquid => BNetwork::Bitcoin, // @FIXME
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidRegtest => BNetwork::Regtest, // @FIXME
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
impl From<BNetwork> for Network {
|
||||
fn from(network: BNetwork) -> Self {
|
||||
impl From<&Network> for B32Network {
|
||||
fn from(network: &Network) -> Self {
|
||||
match network {
|
||||
BNetwork::Bitcoin => Network::Bitcoin,
|
||||
BNetwork::Testnet => Network::Testnet,
|
||||
BNetwork::Testnet4 => Network::Testnet4,
|
||||
BNetwork::Regtest => Network::Regtest,
|
||||
BNetwork::Signet => Network::Signet,
|
||||
Network::Bitcoin => B32Network::Bitcoin,
|
||||
Network::Testnet => B32Network::Testnet,
|
||||
Network::Regtest => B32Network::Regtest,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::Liquid => B32Network::Bitcoin, // @FIXME
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidRegtest => B32Network::Regtest, // @FIXME
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&BNetwork> for Network {
|
||||
fn from(network: &BNetwork) -> Self {
|
||||
match network {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
BNetwork::Bitcoin => Network::Bitcoin,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
BNetwork::Regtest => Network::Regtest,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
BNetwork::Bitcoin => Network::Liquid, // @FIXME
|
||||
#[cfg(feature = "liquid")]
|
||||
BNetwork::Regtest => Network::LiquidRegtest, // @FIXME
|
||||
BNetwork::Testnet => Network::Testnet, // @FIXME
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
481
src/config.rs
481
src/config.rs
@ -1,109 +1,55 @@
|
||||
use clap::{App, Arg};
|
||||
use dirs::home_dir;
|
||||
use num_cpus;
|
||||
use std::fs;
|
||||
use std::net::SocketAddr;
|
||||
use std::net::ToSocketAddrs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::sync::Arc;
|
||||
use stderrlog;
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
use bitcoin_hashes::hex::ToHex;
|
||||
|
||||
use crate::chain::Network;
|
||||
use crate::daemon::CookieGetter;
|
||||
|
||||
use crate::errors::*;
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
use bitcoin::Network as BNetwork;
|
||||
|
||||
pub(crate) const APP_NAME: &str = "mempool-electrs";
|
||||
pub(crate) const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
pub(crate) const GIT_HASH: Option<&str> = option_env!("GIT_HASH");
|
||||
// This will be set only once in the Daemon::new() constructor at startup
|
||||
pub(crate) static BITCOIND_SUBVER: OnceLock<String> = OnceLock::new();
|
||||
|
||||
lazy_static! {
|
||||
pub(crate) static ref VERSION_STRING: String = {
|
||||
if let Some(hash) = GIT_HASH {
|
||||
format!("{} {}-{}", APP_NAME, ELECTRS_VERSION, hash)
|
||||
} else {
|
||||
format!("{} {}", APP_NAME, ELECTRS_VERSION)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Config {
|
||||
// See below for the documentation of each field:
|
||||
pub log: stderrlog::StdErrLog,
|
||||
pub network_type: Network,
|
||||
pub magic: Option<u32>,
|
||||
pub db_path: PathBuf,
|
||||
pub daemon_dir: PathBuf,
|
||||
pub blocks_dir: PathBuf,
|
||||
pub daemon_rpc_addr: SocketAddr,
|
||||
pub cookie: Option<String>,
|
||||
pub electrum_rpc_addr: SocketAddr,
|
||||
pub http_addr: SocketAddr,
|
||||
pub http_socket_file: Option<PathBuf>,
|
||||
pub rpc_socket_file: Option<PathBuf>,
|
||||
pub monitoring_addr: SocketAddr,
|
||||
pub jsonrpc_import: bool,
|
||||
pub light_mode: bool,
|
||||
pub main_loop_delay: u64,
|
||||
pub address_search: bool,
|
||||
pub index_unspendables: bool,
|
||||
pub index_batch_size: usize,
|
||||
pub bulk_index_threads: usize,
|
||||
pub tx_cache_size: usize,
|
||||
pub prevout_enabled: bool,
|
||||
pub cors: Option<String>,
|
||||
pub precache_scripts: Option<String>,
|
||||
pub precache_threads: usize,
|
||||
pub utxos_limit: usize,
|
||||
pub electrum_txs_limit: usize,
|
||||
pub electrum_banner: String,
|
||||
pub mempool_backlog_stats_ttl: u64,
|
||||
pub mempool_recent_txs_size: usize,
|
||||
pub rest_default_block_limit: usize,
|
||||
pub rest_default_chain_txs_per_page: usize,
|
||||
pub rest_default_max_mempool_txs: usize,
|
||||
pub rest_default_max_address_summary_txs: usize,
|
||||
pub rest_max_mempool_page_size: usize,
|
||||
pub rest_max_mempool_txid_page_size: usize,
|
||||
pub electrum_max_line_size: usize,
|
||||
pub electrum_max_subscriptions: usize,
|
||||
pub electrum_max_clients: usize,
|
||||
pub electrum_idle_timeout: u64,
|
||||
pub electrum_haproxy_depth: usize,
|
||||
pub electrum_connections_per_client: usize,
|
||||
pub electrum_public_hosts: Option<crate::electrum::ServerHosts>,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub parent_network: BNetwork,
|
||||
pub parent_network: Network,
|
||||
#[cfg(feature = "liquid")]
|
||||
pub asset_db_path: Option<PathBuf>,
|
||||
|
||||
#[cfg(feature = "electrum-discovery")]
|
||||
pub electrum_announce: bool,
|
||||
#[cfg(feature = "electrum-discovery")]
|
||||
pub tor_proxy: Option<std::net::SocketAddr>,
|
||||
}
|
||||
|
||||
fn str_to_socketaddr(address: &str, what: &str) -> SocketAddr {
|
||||
address
|
||||
.to_socket_addrs()
|
||||
.unwrap_or_else(|_| panic!("unable to resolve {} address", what))
|
||||
.collect::<Vec<_>>()
|
||||
.pop()
|
||||
.unwrap()
|
||||
pub parent_genesis_hash: String,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn from_args() -> Config {
|
||||
let network_help = format!("Select network type ({})", Network::names().join(", "));
|
||||
let network_help = format!(
|
||||
"Select Bitcoin network type ({})",
|
||||
Network::names().join(", ")
|
||||
);
|
||||
|
||||
let args = App::new("Mempool Electrum Rust Server")
|
||||
let args = App::new("Electrum Rust Server")
|
||||
.version(crate_version!())
|
||||
.arg(
|
||||
Arg::with_name("version")
|
||||
.long("version")
|
||||
.help("Print out the version of this app and quit immediately."),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("verbosity")
|
||||
.short("v")
|
||||
@ -127,12 +73,6 @@ impl Config {
|
||||
.help("Data directory of Bitcoind (default: ~/.bitcoin/)")
|
||||
.takes_value(true),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("blocks_dir")
|
||||
.long("blocks-dir")
|
||||
.help("Analogous to bitcoind's -blocksdir option, this specifies the directory containing the raw blocks files (blk*.dat) (default: ~/.bitcoin/blocks/)")
|
||||
.takes_value(true),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("cookie")
|
||||
.long("cookie")
|
||||
@ -145,12 +85,6 @@ impl Config {
|
||||
.help(&network_help)
|
||||
.takes_value(true),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("magic")
|
||||
.long("magic")
|
||||
.default_value("")
|
||||
.takes_value(true),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("electrum_rpc_addr")
|
||||
.long("electrum-rpc-addr")
|
||||
@ -181,25 +115,27 @@ impl Config {
|
||||
.help("Use JSONRPC instead of directly importing blk*.dat files. Useful for remote full node or low memory system"),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("light_mode")
|
||||
.long("lightmode")
|
||||
.help("Enable light mode for reduced storage")
|
||||
Arg::with_name("index_batch_size")
|
||||
.long("index-batch-size")
|
||||
.help("Number of blocks to get in one JSONRPC request from bitcoind")
|
||||
.default_value("100"),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("main_loop_delay")
|
||||
.long("main-loop-delay")
|
||||
.help("The number of milliseconds the main loop will wait between loops. (Can be shortened with SIGUSR1)")
|
||||
.default_value("500")
|
||||
Arg::with_name("bulk_index_threads")
|
||||
.long("bulk-index-threads")
|
||||
.help("Number of threads used for bulk indexing (default: use the # of CPUs)")
|
||||
.default_value("0")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("address_search")
|
||||
.long("address-search")
|
||||
.help("Enable prefix address search")
|
||||
Arg::with_name("tx_cache_size")
|
||||
.long("tx-cache-size")
|
||||
.help("Number of transactions to keep in for query LRU cache")
|
||||
.default_value("10000") // should be enough for a small wallet.
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("index_unspendables")
|
||||
.long("index-unspendables")
|
||||
.help("Enable indexing of provably unspendable outputs")
|
||||
Arg::with_name("disable_prevout")
|
||||
.long("disable-prevout")
|
||||
.help("Don't attach previous output details to inputs")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("cors")
|
||||
@ -212,329 +148,110 @@ impl Config {
|
||||
.long("precache-scripts")
|
||||
.help("Path to file with list of scripts to pre-cache")
|
||||
.takes_value(true)
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("precache_threads")
|
||||
.long("precache-threads")
|
||||
.help("Non-zero number of threads to use for precache threadpool. [default: 4 * CORE_COUNT]")
|
||||
.takes_value(true)
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("utxos_limit")
|
||||
.long("utxos-limit")
|
||||
.help("Maximum number of utxos to process per address. Lookups for addresses with more utxos will fail. Applies to the Electrum and HTTP APIs.")
|
||||
.default_value("500")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("mempool_backlog_stats_ttl")
|
||||
.long("mempool-backlog-stats-ttl")
|
||||
.help("The number of seconds that need to pass before Mempool::update will update the latency histogram again.")
|
||||
.default_value("10")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("mempool_recent_txs_size")
|
||||
.long("mempool-recent-txs-size")
|
||||
.help("The number of transactions that mempool will keep in its recents queue. This is returned by mempool/recent endpoint.")
|
||||
.default_value("10")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("rest_default_block_limit")
|
||||
.long("rest-default-block-limit")
|
||||
.help("The default number of blocks returned from the blocks/[start_height] endpoint.")
|
||||
.default_value("10")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("rest_default_chain_txs_per_page")
|
||||
.long("rest-default-chain-txs-per-page")
|
||||
.help("The default number of on-chain transactions returned by the txs endpoints.")
|
||||
.default_value("25")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("rest_default_max_mempool_txs")
|
||||
.long("rest-default-max-mempool-txs")
|
||||
.help("The default number of mempool transactions returned by the txs endpoints.")
|
||||
.default_value("50")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("rest_default_max_address_summary_txs")
|
||||
.long("rest-default-max-address-summary-txs")
|
||||
.help("The default number of transactions returned by the address summary endpoints.")
|
||||
.default_value("5000")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("rest_max_mempool_page_size")
|
||||
.long("rest-max-mempool-page-size")
|
||||
.help("The maximum number of transactions returned by the paginated /internal/mempool/txs endpoint.")
|
||||
.default_value("1000")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("rest_max_mempool_txid_page_size")
|
||||
.long("rest-max-mempool-txid-page-size")
|
||||
.help("The maximum number of transactions returned by the paginated /mempool/txids/page endpoint.")
|
||||
.default_value("10000")
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("electrum_txs_limit")
|
||||
.long("electrum-txs-limit")
|
||||
.help("Maximum number of transactions returned by Electrum history queries. Lookups with more results will fail.")
|
||||
.default_value("500")
|
||||
).arg(
|
||||
Arg::with_name("electrum_banner")
|
||||
.long("electrum-banner")
|
||||
.help("Welcome banner for the Electrum server, shown in the console to clients.")
|
||||
.takes_value(true)
|
||||
).arg(
|
||||
Arg::with_name("electrum_max_line_size")
|
||||
.long("electrum-max-line-size")
|
||||
.help("Maximum size of a single Electrum request line in bytes (default: 1 MiB).")
|
||||
.default_value("1048576")
|
||||
).arg(
|
||||
Arg::with_name("electrum_max_subscriptions")
|
||||
.long("electrum-max-subscriptions")
|
||||
.help("Maximum number of scripthash subscriptions per client connection.")
|
||||
.default_value("100")
|
||||
).arg(
|
||||
Arg::with_name("electrum_max_clients")
|
||||
.long("electrum-max-clients")
|
||||
.help("Maximum number of concurrent Electrum client connections.")
|
||||
.default_value("10")
|
||||
).arg(
|
||||
Arg::with_name("electrum_idle_timeout")
|
||||
.long("electrum-idle-timeout")
|
||||
.help("Maximum idle time in seconds since the last client request before disconnecting the Electrum connection.")
|
||||
.default_value("600")
|
||||
).arg(
|
||||
Arg::with_name("electrum_haproxy_depth")
|
||||
.long("electrum-haproxy-depth")
|
||||
.help("Which HAProxy PROXY-protocol header layer identifies the real client IP. 0 disables PROXY-protocol detection; 1 uses the first (outermost) address, 2 the second, and so on. If the requested layer or any PROXY header is absent, no client IP is associated with the connection.")
|
||||
.default_value("0")
|
||||
).arg(
|
||||
Arg::with_name("electrum_connections_per_client")
|
||||
.long("electrum-connections-per-client")
|
||||
.help("Maximum number of concurrent Electrum connections allowed per client (keyed by the HAProxy-reported address when available, otherwise the peer IP). 0 disables the per-client limit.")
|
||||
.default_value("10")
|
||||
);
|
||||
|
||||
#[cfg(unix)]
|
||||
let args = args.arg(
|
||||
Arg::with_name("http_socket_file")
|
||||
.long("http-socket-file")
|
||||
.help("HTTP server 'unix socket file' to listen on (default disabled, enabling this disables the http server)")
|
||||
.takes_value(true),
|
||||
);
|
||||
|
||||
#[cfg(unix)]
|
||||
let args = args.arg(
|
||||
Arg::with_name("rpc_socket_file")
|
||||
.long("rpc-socket-file")
|
||||
.help("Electrum RPC 'unix socket file' to listen on (default disabled, enabling this ignores the electrum_rpc_addr arg)")
|
||||
.takes_value(true),
|
||||
);
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
let args = args
|
||||
.arg(
|
||||
Arg::with_name("parent_network")
|
||||
.long("parent-network")
|
||||
.help("Select parent network type (mainnet, testnet, regtest)")
|
||||
.takes_value(true),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("asset_db_path")
|
||||
.long("asset-db-path")
|
||||
.help("Directory for liquid/elements asset db")
|
||||
.takes_value(true),
|
||||
);
|
||||
|
||||
#[cfg(feature = "electrum-discovery")]
|
||||
let args = args.arg(
|
||||
Arg::with_name("electrum_public_hosts")
|
||||
.long("electrum-public-hosts")
|
||||
.help("A dictionary of hosts where the Electrum server can be reached at. Required to enable server discovery. See https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server-features")
|
||||
.takes_value(true)
|
||||
).arg(
|
||||
Arg::with_name("electrum_announce")
|
||||
.long("electrum-announce")
|
||||
.help("Announce the Electrum server to other servers")
|
||||
).arg(
|
||||
Arg::with_name("tor_proxy")
|
||||
.long("tor-proxy")
|
||||
.help("ip:addr of socks proxy for accessing onion hosts")
|
||||
Arg::with_name("parent_network")
|
||||
.long("parent-network")
|
||||
.help("Select parent network type (mainnet, testnet, regtest)")
|
||||
.takes_value(true),
|
||||
);
|
||||
|
||||
let m = args.get_matches();
|
||||
|
||||
if m.is_present("version") {
|
||||
eprintln!("{}", *VERSION_STRING);
|
||||
std::process::exit(0);
|
||||
}
|
||||
|
||||
let network_name = m.value_of("network").unwrap_or("mainnet");
|
||||
let network_type = Network::from(network_name);
|
||||
let magic: Option<u32> = m
|
||||
.value_of("magic")
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| u32::from_str_radix(s, 16).expect("invalid network magic"));
|
||||
let db_dir = Path::new(m.value_of("db_dir").unwrap_or("./db"));
|
||||
let db_path = db_dir.join(network_name);
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
let parent_network = m
|
||||
.value_of("parent_network")
|
||||
.map(|s| s.parse().expect("invalid parent network"))
|
||||
.unwrap_or_else(|| match network_type {
|
||||
Network::Liquid => BNetwork::Bitcoin,
|
||||
// XXX liquid testnet/regtest don't have a parent chain
|
||||
Network::LiquidTestnet | Network::LiquidRegtest => BNetwork::Regtest,
|
||||
});
|
||||
|
||||
let parent_network = Network::from(m.value_of("parent_network").unwrap_or("mainnet"));
|
||||
#[cfg(feature = "liquid")]
|
||||
let asset_db_path = m.value_of("asset_db_path").map(PathBuf::from);
|
||||
let parent_genesis_hash = parent_network.genesis_hash().to_hex();
|
||||
|
||||
let default_daemon_port = match network_type {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Bitcoin => 8332,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Testnet => 18332,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Regtest => 18443,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Signet => 38332,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Testnet4 => 48332,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::Liquid => 7041,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidTestnet | Network::LiquidRegtest => 7040,
|
||||
Network::LiquidRegtest => 7041,
|
||||
};
|
||||
let default_electrum_port = match network_type {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Bitcoin => 50001,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Testnet => 60001,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Testnet4 => 40001,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Regtest => 60401,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Signet => 60601,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::Liquid => 51000,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidTestnet => 51301,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidRegtest => 51401,
|
||||
};
|
||||
let default_http_port = match network_type {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Bitcoin => 3000,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Testnet => 3001,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Regtest => 3002,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Signet => 3003,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Testnet4 => 3004,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::Liquid => 3000,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidTestnet => 3001,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidRegtest => 3002,
|
||||
};
|
||||
let default_monitoring_port = match network_type {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Bitcoin => 4224,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Testnet => 14224,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Regtest => 24224,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Testnet4 => 44224,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Signet => 54224,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::Liquid => 34224,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidTestnet => 44324,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidRegtest => 44224,
|
||||
};
|
||||
|
||||
let daemon_rpc_addr: SocketAddr = str_to_socketaddr(
|
||||
m.value_of("daemon_rpc_addr")
|
||||
.unwrap_or(&format!("127.0.0.1:{}", default_daemon_port)),
|
||||
"Bitcoin RPC",
|
||||
);
|
||||
let electrum_rpc_addr: SocketAddr = str_to_socketaddr(
|
||||
m.value_of("electrum_rpc_addr")
|
||||
.unwrap_or(&format!("127.0.0.1:{}", default_electrum_port)),
|
||||
"Electrum RPC",
|
||||
);
|
||||
let http_addr: SocketAddr = str_to_socketaddr(
|
||||
m.value_of("http_addr")
|
||||
.unwrap_or(&format!("127.0.0.1:{}", default_http_port)),
|
||||
"HTTP Server",
|
||||
);
|
||||
|
||||
let http_socket_file: Option<PathBuf> = m.value_of("http_socket_file").map(PathBuf::from);
|
||||
let rpc_socket_file: Option<PathBuf> = m.value_of("rpc_socket_file").map(PathBuf::from);
|
||||
let monitoring_addr: SocketAddr = str_to_socketaddr(
|
||||
m.value_of("monitoring_addr")
|
||||
.unwrap_or(&format!("127.0.0.1:{}", default_monitoring_port)),
|
||||
"Prometheus monitoring",
|
||||
);
|
||||
let daemon_rpc_addr: SocketAddr = m
|
||||
.value_of("daemon_rpc_addr")
|
||||
.unwrap_or(&format!("127.0.0.1:{}", default_daemon_port))
|
||||
.parse()
|
||||
.expect("invalid Bitcoind RPC address");
|
||||
let electrum_rpc_addr: SocketAddr = m
|
||||
.value_of("electrum_rpc_addr")
|
||||
.unwrap_or(&format!("127.0.0.1:{}", default_electrum_port))
|
||||
.parse()
|
||||
.expect("invalid Electrum RPC address");
|
||||
let http_addr: SocketAddr = m
|
||||
.value_of("http_addr")
|
||||
.unwrap_or(&format!("127.0.0.1:{}", default_http_port))
|
||||
.parse()
|
||||
.expect("invalid HTTP server address");
|
||||
let monitoring_addr: SocketAddr = m
|
||||
.value_of("monitoring_addr")
|
||||
.unwrap_or(&format!("127.0.0.1:{}", default_monitoring_port))
|
||||
.parse()
|
||||
.expect("invalid Prometheus monitoring address");
|
||||
|
||||
let mut daemon_dir = m
|
||||
.value_of("daemon_dir")
|
||||
.map(PathBuf::from)
|
||||
.map(|p| PathBuf::from(p))
|
||||
.unwrap_or_else(|| {
|
||||
let mut default_dir = home_dir().expect("no homedir");
|
||||
default_dir.push(".bitcoin");
|
||||
default_dir
|
||||
});
|
||||
match network_type {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Bitcoin => (),
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Testnet => daemon_dir.push("testnet3"),
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Testnet4 => daemon_dir.push("testnet4"),
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Regtest => daemon_dir.push("regtest"),
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Signet => daemon_dir.push("signet"),
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::Liquid => daemon_dir.push("liquidv1"),
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidTestnet => daemon_dir.push("liquidtestnet"),
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidRegtest => daemon_dir.push("liquidregtest"),
|
||||
}
|
||||
let blocks_dir = m
|
||||
.value_of("blocks_dir")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| daemon_dir.join("blocks"));
|
||||
let cookie = m.value_of("cookie").map(|s| s.to_owned());
|
||||
|
||||
let electrum_banner = m
|
||||
.value_of("electrum_banner")
|
||||
.map_or_else(|| format!("Welcome to {}", *VERSION_STRING), |s| s.into());
|
||||
|
||||
#[cfg(feature = "electrum-discovery")]
|
||||
let electrum_public_hosts = m
|
||||
.value_of("electrum_public_hosts")
|
||||
.map(|s| serde_json::from_str(s).expect("invalid --electrum-public-hosts"));
|
||||
#[cfg(not(feature = "electrum-discovery"))]
|
||||
let electrum_public_hosts: Option<crate::electrum::ServerHosts> = None;
|
||||
|
||||
let mut log = stderrlog::new();
|
||||
log.verbosity(m.occurrences_of("verbosity") as usize);
|
||||
log.timestamp(if m.is_present("timestamp") {
|
||||
@ -543,97 +260,37 @@ impl Config {
|
||||
stderrlog::Timestamp::Off
|
||||
});
|
||||
log.init().expect("logging initialization failed");
|
||||
let mut bulk_index_threads = value_t_or_exit!(m, "bulk_index_threads", usize);
|
||||
if bulk_index_threads == 0 {
|
||||
bulk_index_threads = num_cpus::get();
|
||||
}
|
||||
let config = Config {
|
||||
log,
|
||||
network_type,
|
||||
magic,
|
||||
db_path,
|
||||
daemon_dir,
|
||||
blocks_dir,
|
||||
daemon_rpc_addr,
|
||||
cookie,
|
||||
utxos_limit: value_t_or_exit!(m, "utxos_limit", usize),
|
||||
electrum_rpc_addr,
|
||||
electrum_txs_limit: value_t_or_exit!(m, "electrum_txs_limit", usize),
|
||||
electrum_banner,
|
||||
http_addr,
|
||||
http_socket_file,
|
||||
rpc_socket_file,
|
||||
monitoring_addr,
|
||||
mempool_backlog_stats_ttl: value_t_or_exit!(m, "mempool_backlog_stats_ttl", u64),
|
||||
mempool_recent_txs_size: value_t_or_exit!(m, "mempool_recent_txs_size", usize),
|
||||
rest_default_block_limit: value_t_or_exit!(m, "rest_default_block_limit", usize),
|
||||
rest_default_chain_txs_per_page: value_t_or_exit!(
|
||||
m,
|
||||
"rest_default_chain_txs_per_page",
|
||||
usize
|
||||
),
|
||||
rest_default_max_mempool_txs: value_t_or_exit!(
|
||||
m,
|
||||
"rest_default_max_mempool_txs",
|
||||
usize
|
||||
),
|
||||
rest_default_max_address_summary_txs: value_t_or_exit!(
|
||||
m,
|
||||
"rest_default_max_address_summary_txs",
|
||||
usize
|
||||
),
|
||||
rest_max_mempool_page_size: value_t_or_exit!(m, "rest_max_mempool_page_size", usize),
|
||||
rest_max_mempool_txid_page_size: value_t_or_exit!(
|
||||
m,
|
||||
"rest_max_mempool_txid_page_size",
|
||||
usize
|
||||
),
|
||||
electrum_max_line_size: value_t_or_exit!(m, "electrum_max_line_size", usize),
|
||||
electrum_max_subscriptions: value_t_or_exit!(m, "electrum_max_subscriptions", usize),
|
||||
electrum_max_clients: value_t_or_exit!(m, "electrum_max_clients", usize),
|
||||
electrum_idle_timeout: value_t_or_exit!(m, "electrum_idle_timeout", u64),
|
||||
electrum_haproxy_depth: value_t_or_exit!(m, "electrum_haproxy_depth", usize),
|
||||
electrum_connections_per_client: value_t_or_exit!(
|
||||
m,
|
||||
"electrum_connections_per_client",
|
||||
usize
|
||||
),
|
||||
jsonrpc_import: m.is_present("jsonrpc_import"),
|
||||
light_mode: m.is_present("light_mode"),
|
||||
main_loop_delay: value_t_or_exit!(m, "main_loop_delay", u64),
|
||||
address_search: m.is_present("address_search"),
|
||||
index_unspendables: m.is_present("index_unspendables"),
|
||||
index_batch_size: value_t_or_exit!(m, "index_batch_size", usize),
|
||||
bulk_index_threads,
|
||||
tx_cache_size: value_t_or_exit!(m, "tx_cache_size", usize),
|
||||
prevout_enabled: !m.is_present("disable_prevout"),
|
||||
cors: m.value_of("cors").map(|s| s.to_string()),
|
||||
precache_scripts: m.value_of("precache_scripts").map(|s| s.to_string()),
|
||||
precache_threads: m.value_of("precache_threads").map_or_else(
|
||||
|| {
|
||||
std::thread::available_parallelism()
|
||||
.expect("Can't get core count")
|
||||
.get()
|
||||
* 4
|
||||
},
|
||||
|s| match s.parse::<usize>() {
|
||||
Ok(v) if v > 0 => v,
|
||||
_ => clap::Error::value_validation_auto(format!(
|
||||
"The argument '{}' isn't a valid value",
|
||||
s
|
||||
))
|
||||
.exit(),
|
||||
},
|
||||
),
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
parent_network,
|
||||
#[cfg(feature = "liquid")]
|
||||
asset_db_path,
|
||||
|
||||
electrum_public_hosts,
|
||||
#[cfg(feature = "electrum-discovery")]
|
||||
electrum_announce: m.is_present("electrum_announce"),
|
||||
#[cfg(feature = "electrum-discovery")]
|
||||
tor_proxy: m.value_of("tor_proxy").map(|s| s.parse().unwrap()),
|
||||
parent_genesis_hash,
|
||||
};
|
||||
eprintln!("{:?}", config);
|
||||
config
|
||||
}
|
||||
|
||||
pub fn cookie_getter(&self) -> Arc<dyn CookieGetter> {
|
||||
pub fn cookie_getter(&self) -> Arc<CookieGetter> {
|
||||
if let Some(ref value) = self.cookie {
|
||||
Arc::new(StaticCookie {
|
||||
value: value.as_bytes().to_vec(),
|
||||
|
||||
449
src/daemon.rs
449
src/daemon.rs
@ -1,41 +1,32 @@
|
||||
use base64;
|
||||
use bitcoin::consensus::encode::{deserialize, serialize};
|
||||
use bitcoin::util::hash::BitcoinHash;
|
||||
use bitcoin_hashes::hex::{FromHex, ToHex};
|
||||
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
|
||||
use glob;
|
||||
use hex;
|
||||
use serde_json::{from_str, from_value, Value};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::{BufRead, BufReader, Lines, Write};
|
||||
use std::net::{SocketAddr, TcpStream};
|
||||
use std::path::PathBuf;
|
||||
use std::str::FromStr;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use base64;
|
||||
use bitcoin::hashes::Hash;
|
||||
use glob;
|
||||
use hex;
|
||||
use itertools::Itertools;
|
||||
use serde_json::{from_str, from_value, Value};
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
use bitcoin::consensus::encode::{deserialize, serialize};
|
||||
#[cfg(feature = "liquid")]
|
||||
use elements::encode::{deserialize, serialize};
|
||||
|
||||
use crate::chain::{Block, BlockHash, BlockHeader, Network, Transaction, Txid};
|
||||
use crate::config::BITCOIND_SUBVER;
|
||||
use crate::chain::{Block, BlockHeader, Network, Transaction};
|
||||
use crate::metrics::{HistogramOpts, HistogramVec, Metrics};
|
||||
use crate::signal::Waiter;
|
||||
use crate::util::HeaderList;
|
||||
|
||||
use crate::errors::*;
|
||||
|
||||
fn parse_hash<T>(value: &Value) -> Result<T>
|
||||
where
|
||||
T: FromStr,
|
||||
<T as FromStr>::Err: std::fmt::Debug,
|
||||
{
|
||||
value
|
||||
.as_str()
|
||||
.chain_err(|| format!("non-string value: {}", value))?
|
||||
.parse::<T>()
|
||||
.map_err(|e| format!("failed to parse hash: {:?}", e).into())
|
||||
fn parse_hash(value: &Value) -> Result<Sha256dHash> {
|
||||
Ok(Sha256dHash::from_hex(
|
||||
value
|
||||
.as_str()
|
||||
.chain_err(|| format!("non-string value: {}", value))?,
|
||||
)
|
||||
.chain_err(|| format!("non-hex value: {}", value))?)
|
||||
}
|
||||
|
||||
fn header_from_value(value: Value) -> Result<BlockHeader> {
|
||||
@ -43,19 +34,22 @@ fn header_from_value(value: Value) -> Result<BlockHeader> {
|
||||
.as_str()
|
||||
.chain_err(|| format!("non-string header: {}", value))?;
|
||||
let header_bytes = hex::decode(header_hex).chain_err(|| "non-hex header")?;
|
||||
deserialize(&header_bytes).chain_err(|| format!("failed to parse header {}", header_hex))
|
||||
Ok(
|
||||
deserialize(&header_bytes)
|
||||
.chain_err(|| format!("failed to parse header {}", header_hex))?,
|
||||
)
|
||||
}
|
||||
|
||||
fn block_from_value(value: Value) -> Result<Block> {
|
||||
let block_hex = value.as_str().chain_err(|| "non-string block")?;
|
||||
let block_bytes = hex::decode(block_hex).chain_err(|| "non-hex block")?;
|
||||
deserialize(&block_bytes).chain_err(|| format!("failed to parse block {}", block_hex))
|
||||
Ok(deserialize(&block_bytes).chain_err(|| format!("failed to parse block {}", block_hex))?)
|
||||
}
|
||||
|
||||
fn tx_from_value(value: Value) -> Result<Transaction> {
|
||||
let tx_hex = value.as_str().chain_err(|| "non-string tx")?;
|
||||
let tx_bytes = hex::decode(tx_hex).chain_err(|| "non-hex tx")?;
|
||||
deserialize(&tx_bytes).chain_err(|| format!("failed to parse tx {}", tx_hex))
|
||||
Ok(deserialize(&tx_bytes).chain_err(|| format!("failed to parse tx {}", tx_hex))?)
|
||||
}
|
||||
|
||||
/// Parse JSONRPC error code, if exists.
|
||||
@ -67,13 +61,14 @@ fn parse_jsonrpc_reply(mut reply: Value, method: &str, expected_id: u64) -> Resu
|
||||
if let Some(reply_obj) = reply.as_object_mut() {
|
||||
if let Some(err) = reply_obj.get("error") {
|
||||
if !err.is_null() {
|
||||
if let Some(code) = parse_error_code(err) {
|
||||
if let Some(code) = parse_error_code(&err) {
|
||||
match code {
|
||||
// RPC_IN_WARMUP -> retry by later reconnection
|
||||
-28 => bail!(ErrorKind::Connection(err.to_string())),
|
||||
_ => bail!("{} RPC error: {}", method, err),
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
bail!("{} RPC error: {}", method, err);
|
||||
}
|
||||
}
|
||||
let id = reply_obj
|
||||
@ -107,64 +102,38 @@ pub struct BlockchainInfo {
|
||||
pub initialblockdownload: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct MempoolInfo {
|
||||
pub loaded: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
struct NetworkInfo {
|
||||
version: u64,
|
||||
subversion: String,
|
||||
relayfee: f64, // in BTC/kB
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
struct MempoolFees {
|
||||
base: f64,
|
||||
#[serde(rename = "effective-feerate")]
|
||||
effective_feerate: f64,
|
||||
#[serde(rename = "effective-includes")]
|
||||
effective_includes: Vec<String>,
|
||||
pub struct MempoolEntry {
|
||||
fee: u64, // in satoshis
|
||||
vsize: u32, // in virtual bytes (= weight/4)
|
||||
fee_per_vbyte: f32,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct MempoolAcceptResult {
|
||||
txid: String,
|
||||
wtxid: String,
|
||||
allowed: Option<bool>,
|
||||
vsize: Option<u32>,
|
||||
fees: Option<MempoolFees>,
|
||||
#[serde(rename = "reject-reason")]
|
||||
reject_reason: Option<String>,
|
||||
}
|
||||
impl MempoolEntry {
|
||||
fn new(fee: u64, vsize: u32) -> MempoolEntry {
|
||||
MempoolEntry {
|
||||
fee,
|
||||
vsize,
|
||||
fee_per_vbyte: fee as f32 / vsize as f32,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
struct MempoolFeesSubmitPackage {
|
||||
base: f64,
|
||||
#[serde(rename = "effective-feerate")]
|
||||
effective_feerate: Option<f64>,
|
||||
#[serde(rename = "effective-includes")]
|
||||
effective_includes: Option<Vec<String>>,
|
||||
}
|
||||
pub fn fee_per_vbyte(&self) -> f32 {
|
||||
self.fee_per_vbyte
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct SubmitPackageResult {
|
||||
package_msg: String,
|
||||
#[serde(rename = "tx-results")]
|
||||
tx_results: HashMap<String, TxResult>,
|
||||
#[serde(rename = "replaced-transactions")]
|
||||
replaced_transactions: Option<Vec<String>>,
|
||||
}
|
||||
pub fn fee(&self) -> u64 {
|
||||
self.fee
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct TxResult {
|
||||
txid: String,
|
||||
#[serde(rename = "other-wtxid")]
|
||||
other_wtxid: Option<String>,
|
||||
vsize: Option<u32>,
|
||||
fees: Option<MempoolFeesSubmitPackage>,
|
||||
error: Option<String>,
|
||||
pub fn vsize(&self) -> u32 {
|
||||
self.vsize
|
||||
}
|
||||
}
|
||||
|
||||
pub trait CookieGetter: Send + Sync {
|
||||
@ -174,7 +143,7 @@ pub trait CookieGetter: Send + Sync {
|
||||
struct Connection {
|
||||
tx: TcpStream,
|
||||
rx: Lines<BufReader<TcpStream>>,
|
||||
cookie_getter: Arc<dyn CookieGetter>,
|
||||
cookie_getter: Arc<CookieGetter>,
|
||||
addr: SocketAddr,
|
||||
signal: Waiter,
|
||||
}
|
||||
@ -185,7 +154,7 @@ fn tcp_connect(addr: SocketAddr, signal: &Waiter) -> Result<TcpStream> {
|
||||
Ok(conn) => return Ok(conn),
|
||||
Err(err) => {
|
||||
warn!("failed to connect daemon at {}: {}", addr, err);
|
||||
signal.wait(Duration::from_secs(3), false)?;
|
||||
signal.wait(Duration::from_secs(3))?;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@ -195,7 +164,7 @@ fn tcp_connect(addr: SocketAddr, signal: &Waiter) -> Result<TcpStream> {
|
||||
impl Connection {
|
||||
fn new(
|
||||
addr: SocketAddr,
|
||||
cookie_getter: Arc<dyn CookieGetter>,
|
||||
cookie_getter: Arc<CookieGetter>,
|
||||
signal: Waiter,
|
||||
) -> Result<Connection> {
|
||||
let conn = tcp_connect(addr, &signal)?;
|
||||
@ -239,7 +208,7 @@ impl Connection {
|
||||
.chain_err(|| {
|
||||
ErrorKind::Connection("disconnected from daemon while receiving".to_owned())
|
||||
})?
|
||||
.chain_err(|| ErrorKind::Connection("failed to read status".to_owned()))?;
|
||||
.chain_err(|| "failed to read status")?;
|
||||
let mut headers = HashMap::new();
|
||||
for line in iter {
|
||||
let line = line.chain_err(|| ErrorKind::Connection("failed to read".to_owned()))?;
|
||||
@ -312,9 +281,7 @@ impl Counter {
|
||||
|
||||
pub struct Daemon {
|
||||
daemon_dir: PathBuf,
|
||||
blocks_dir: PathBuf,
|
||||
network: Network,
|
||||
magic: Option<u32>,
|
||||
conn: Mutex<Connection>,
|
||||
message_id: Counter, // for monotonic JSONRPC 'id'
|
||||
signal: Waiter,
|
||||
@ -325,22 +292,17 @@ pub struct Daemon {
|
||||
}
|
||||
|
||||
impl Daemon {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
daemon_dir: PathBuf,
|
||||
blocks_dir: PathBuf,
|
||||
daemon_dir: &PathBuf,
|
||||
daemon_rpc_addr: SocketAddr,
|
||||
cookie_getter: Arc<dyn CookieGetter>,
|
||||
cookie_getter: Arc<CookieGetter>,
|
||||
network: Network,
|
||||
magic: Option<u32>,
|
||||
signal: Waiter,
|
||||
metrics: &Metrics,
|
||||
) -> Result<Daemon> {
|
||||
let daemon = Daemon {
|
||||
daemon_dir,
|
||||
blocks_dir,
|
||||
daemon_dir: daemon_dir.clone(),
|
||||
network,
|
||||
magic,
|
||||
conn: Mutex::new(Connection::new(
|
||||
daemon_rpc_addr,
|
||||
cookie_getter,
|
||||
@ -359,42 +321,32 @@ impl Daemon {
|
||||
};
|
||||
let network_info = daemon.getnetworkinfo()?;
|
||||
info!("{:?}", network_info);
|
||||
if network_info.version < 16_00_00 {
|
||||
if network_info.version < 00_16_00_00 {
|
||||
bail!(
|
||||
"{} is not supported - please use bitcoind 0.16+",
|
||||
network_info.subversion,
|
||||
)
|
||||
}
|
||||
// Insert the subversion (/Satoshi xx.xx.xx(comment)/) string from bitcoind
|
||||
_ = BITCOIND_SUBVER.set(network_info.subversion);
|
||||
|
||||
let blockchain_info = daemon.getblockchaininfo()?;
|
||||
info!("{:?}", blockchain_info);
|
||||
if blockchain_info.pruned {
|
||||
if blockchain_info.pruned == true {
|
||||
bail!("pruned node is not supported (use '-prune=0' bitcoind flag)".to_owned())
|
||||
}
|
||||
loop {
|
||||
let info = daemon.getblockchaininfo()?;
|
||||
let mempool = daemon.getmempoolinfo()?;
|
||||
|
||||
let ibd_done = if network.is_regtest() {
|
||||
info.blocks == info.headers
|
||||
} else {
|
||||
!info.initialblockdownload.unwrap_or(false)
|
||||
// initialblockdownload is unavailable on the 0.14-based elements
|
||||
let synced = match info.initialblockdownload {
|
||||
Some(ibd) => !ibd,
|
||||
None => info.verificationprogress == 1.0,
|
||||
};
|
||||
|
||||
if mempool.loaded && ibd_done && info.blocks == info.headers {
|
||||
if synced {
|
||||
break;
|
||||
}
|
||||
|
||||
warn!(
|
||||
"waiting for bitcoind sync and mempool load to finish: {}/{} blocks, verification progress: {:.3}%, mempool loaded: {}",
|
||||
info.blocks,
|
||||
info.headers,
|
||||
info.verificationprogress * 100.0,
|
||||
mempool.loaded
|
||||
);
|
||||
signal.wait(Duration::from_secs(5), false)?;
|
||||
warn!("wait until bitcoind is synced");
|
||||
signal.wait(Duration::from_secs(3))?;
|
||||
}
|
||||
Ok(daemon)
|
||||
}
|
||||
@ -402,9 +354,7 @@ impl Daemon {
|
||||
pub fn reconnect(&self) -> Result<Daemon> {
|
||||
Ok(Daemon {
|
||||
daemon_dir: self.daemon_dir.clone(),
|
||||
blocks_dir: self.blocks_dir.clone(),
|
||||
network: self.network,
|
||||
magic: self.magic,
|
||||
conn: Mutex::new(self.conn.lock().unwrap().reconnect()?),
|
||||
message_id: Counter::new(),
|
||||
signal: self.signal.clone(),
|
||||
@ -414,7 +364,9 @@ impl Daemon {
|
||||
}
|
||||
|
||||
pub fn list_blk_files(&self) -> Result<Vec<PathBuf>> {
|
||||
let path = self.blocks_dir.join("blk*.dat");
|
||||
let mut path = self.daemon_dir.clone();
|
||||
path.push("blocks");
|
||||
path.push("blk*.dat");
|
||||
debug!("listing block files at {:?}", path);
|
||||
let mut paths: Vec<PathBuf> = glob::glob(path.to_str().unwrap())
|
||||
.chain_err(|| "failed to list blk*.dat files")?
|
||||
@ -425,7 +377,7 @@ impl Daemon {
|
||||
}
|
||||
|
||||
pub fn magic(&self) -> u32 {
|
||||
self.magic.unwrap_or_else(|| self.network.magic())
|
||||
self.network.magic()
|
||||
}
|
||||
|
||||
fn call_jsonrpc(&self, method: &str, request: &Value) -> Result<Value> {
|
||||
@ -445,66 +397,29 @@ impl Daemon {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn handle_request_batch(
|
||||
&self,
|
||||
method: &str,
|
||||
params_list: &[Value],
|
||||
failure_threshold: f64,
|
||||
) -> Result<Vec<Value>> {
|
||||
fn handle_request_batch(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
|
||||
let id = self.message_id.next();
|
||||
let chunks = params_list
|
||||
let reqs = params_list
|
||||
.iter()
|
||||
.map(|params| json!({"method": method, "params": params, "id": id}))
|
||||
.chunks(50_000); // Max Amount of batched requests
|
||||
.collect();
|
||||
let mut results = vec![];
|
||||
let total_requests = params_list.len();
|
||||
let mut failed_requests: u64 = 0;
|
||||
let threshold = (failure_threshold * total_requests as f64).round() as u64;
|
||||
let mut n = 0;
|
||||
|
||||
for chunk in &chunks {
|
||||
let reqs = chunk.collect();
|
||||
let mut replies = self.call_jsonrpc(method, &reqs)?;
|
||||
if let Some(replies_vec) = replies.as_array_mut() {
|
||||
for reply in replies_vec {
|
||||
n += 1;
|
||||
match parse_jsonrpc_reply(reply.take(), method, id) {
|
||||
Ok(parsed_reply) => results.push(parsed_reply),
|
||||
Err(e) => {
|
||||
failed_requests += 1;
|
||||
warn!(
|
||||
"batch request {} {}/{} failed: {}",
|
||||
method,
|
||||
n,
|
||||
total_requests,
|
||||
e.to_string()
|
||||
);
|
||||
// abort and return the last error once a threshold number of requests have failed
|
||||
if failed_requests > threshold {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
bail!("non-array replies: {:?}", replies);
|
||||
let mut replies = self.call_jsonrpc(method, &reqs)?;
|
||||
if let Some(replies_vec) = replies.as_array_mut() {
|
||||
for reply in replies_vec {
|
||||
results.push(parse_jsonrpc_reply(reply.take(), method, id)?)
|
||||
}
|
||||
return Ok(results);
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
bail!("non-array replies: {:?}", replies);
|
||||
}
|
||||
|
||||
fn retry_request_batch(
|
||||
&self,
|
||||
method: &str,
|
||||
params_list: &[Value],
|
||||
failure_threshold: f64,
|
||||
) -> Result<Vec<Value>> {
|
||||
fn retry_request_batch(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
|
||||
loop {
|
||||
match self.handle_request_batch(method, params_list, failure_threshold) {
|
||||
match self.handle_request_batch(method, params_list) {
|
||||
Err(Error(ErrorKind::Connection(msg), _)) => {
|
||||
warn!("reconnecting to bitcoind: {}", msg);
|
||||
self.signal.wait(Duration::from_secs(3), false)?;
|
||||
self.signal.wait(Duration::from_secs(3))?;
|
||||
let mut conn = self.conn.lock().unwrap();
|
||||
*conn = conn.reconnect()?;
|
||||
continue;
|
||||
@ -515,40 +430,35 @@ impl Daemon {
|
||||
}
|
||||
|
||||
fn request(&self, method: &str, params: Value) -> Result<Value> {
|
||||
let mut values = self.retry_request_batch(method, &[params], 0.0)?;
|
||||
let mut values = self.retry_request_batch(method, &[params])?;
|
||||
assert_eq!(values.len(), 1);
|
||||
Ok(values.remove(0))
|
||||
}
|
||||
|
||||
fn requests(&self, method: &str, params_list: &[Value]) -> Result<Vec<Value>> {
|
||||
self.retry_request_batch(method, params_list, 0.0)
|
||||
self.retry_request_batch(method, params_list)
|
||||
}
|
||||
|
||||
// bitcoind JSONRPC API:
|
||||
|
||||
pub fn getblockchaininfo(&self) -> Result<BlockchainInfo> {
|
||||
let info: Value = self.request("getblockchaininfo", json!([]))?;
|
||||
from_value(info).chain_err(|| "invalid blockchain info")
|
||||
}
|
||||
|
||||
fn getmempoolinfo(&self) -> Result<MempoolInfo> {
|
||||
let info: Value = self.request("getmempoolinfo", json!([]))?;
|
||||
from_value(info).chain_err(|| "invalid mempool info")
|
||||
Ok(from_value(info).chain_err(|| "invalid blockchain info")?)
|
||||
}
|
||||
|
||||
fn getnetworkinfo(&self) -> Result<NetworkInfo> {
|
||||
let info: Value = self.request("getnetworkinfo", json!([]))?;
|
||||
from_value(info).chain_err(|| "invalid network info")
|
||||
Ok(from_value(info).chain_err(|| "invalid network info")?)
|
||||
}
|
||||
|
||||
pub fn getbestblockhash(&self) -> Result<BlockHash> {
|
||||
parse_hash(&self.request("getbestblockhash", json!([]))?)
|
||||
pub fn getbestblockhash(&self) -> Result<Sha256dHash> {
|
||||
parse_hash(&self.request("getbestblockhash", json!([]))?).chain_err(|| "invalid blockhash")
|
||||
}
|
||||
|
||||
pub fn getblockheader(&self, blockhash: &BlockHash) -> Result<BlockHeader> {
|
||||
pub fn getblockheader(&self, blockhash: &Sha256dHash) -> Result<BlockHeader> {
|
||||
header_from_value(self.request(
|
||||
"getblockheader",
|
||||
json!([blockhash.to_string(), /*verbose=*/ false]),
|
||||
json!([blockhash.to_hex(), /*verbose=*/ false]),
|
||||
)?)
|
||||
}
|
||||
|
||||
@ -566,23 +476,22 @@ impl Daemon {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn getblock(&self, blockhash: &BlockHash) -> Result<Block> {
|
||||
let block = block_from_value(self.request(
|
||||
"getblock",
|
||||
json!([blockhash.to_string(), /*verbose=*/ false]),
|
||||
)?)?;
|
||||
assert_eq!(block.block_hash(), *blockhash);
|
||||
pub fn getblock(&self, blockhash: &Sha256dHash) -> Result<Block> {
|
||||
let block = block_from_value(
|
||||
self.request("getblock", json!([blockhash.to_hex(), /*verbose=*/ false]))?,
|
||||
)?;
|
||||
assert_eq!(block.bitcoin_hash(), *blockhash);
|
||||
Ok(block)
|
||||
}
|
||||
|
||||
pub fn getblock_raw(&self, blockhash: &BlockHash, verbose: u32) -> Result<Value> {
|
||||
self.request("getblock", json!([blockhash.to_string(), verbose]))
|
||||
pub fn getblock_raw(&self, blockhash: &Sha256dHash, verbose: u32) -> Result<Value> {
|
||||
self.request("getblock", json!([blockhash.to_hex(), verbose]))
|
||||
}
|
||||
|
||||
pub fn getblocks(&self, blockhashes: &[BlockHash]) -> Result<Vec<Block>> {
|
||||
pub fn getblocks(&self, blockhashes: &[Sha256dHash]) -> Result<Vec<Block>> {
|
||||
let params_list: Vec<Value> = blockhashes
|
||||
.iter()
|
||||
.map(|hash| json!([hash.to_string(), /*verbose=*/ false]))
|
||||
.map(|hash| json!([hash.to_hex(), /*verbose=*/ false]))
|
||||
.collect();
|
||||
let values = self.requests("getblock", ¶ms_list)?;
|
||||
let mut blocks = vec![];
|
||||
@ -592,145 +501,104 @@ impl Daemon {
|
||||
Ok(blocks)
|
||||
}
|
||||
|
||||
pub fn gettransactions(&self, txhashes: &[&Txid]) -> Result<Vec<Transaction>> {
|
||||
pub fn gettransactions(&self, txhashes: &[&Sha256dHash]) -> Result<Vec<Transaction>> {
|
||||
let params_list: Vec<Value> = txhashes
|
||||
.iter()
|
||||
.map(|txhash| json!([txhash.to_string(), /*verbose=*/ false]))
|
||||
.map(|txhash| json!([txhash.to_hex(), /*verbose=*/ false]))
|
||||
.collect();
|
||||
let values = self.retry_request_batch("getrawtransaction", ¶ms_list, 0.25)?;
|
||||
|
||||
let values = self.requests("getrawtransaction", ¶ms_list)?;
|
||||
let mut txs = vec![];
|
||||
for value in values {
|
||||
txs.push(tx_from_value(value)?);
|
||||
}
|
||||
// missing transactions are skipped, so the number of txs returned may be less than the number of txids requested
|
||||
assert_eq!(txhashes.len(), txs.len());
|
||||
Ok(txs)
|
||||
}
|
||||
|
||||
pub fn gettransaction_raw(
|
||||
&self,
|
||||
txid: &Txid,
|
||||
blockhash: &BlockHash,
|
||||
verbose: bool,
|
||||
) -> Result<Value> {
|
||||
self.request(
|
||||
"getrawtransaction",
|
||||
json!([txid.to_string(), verbose, blockhash]),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn getmempooltx(&self, txhash: &Txid) -> Result<Transaction> {
|
||||
pub fn getmempooltx(&self, txhash: &Sha256dHash) -> Result<Transaction> {
|
||||
let value = self.request(
|
||||
"getrawtransaction",
|
||||
json!([txhash.to_string(), /*verbose=*/ false]),
|
||||
json!([txhash.to_hex(), /*verbose=*/ false]),
|
||||
)?;
|
||||
tx_from_value(value)
|
||||
}
|
||||
|
||||
pub fn getmempooltxids(&self) -> Result<HashSet<Txid>> {
|
||||
let res = self.request("getrawmempool", json!([/*verbose=*/ false]))?;
|
||||
serde_json::from_value(res).chain_err(|| "invalid getrawmempool reply")
|
||||
pub fn getmempooltxids(&self) -> Result<HashSet<Sha256dHash>> {
|
||||
let txids: Value = self.request("getrawmempool", json!([/*verbose=*/ false]))?;
|
||||
let mut result = HashSet::new();
|
||||
for value in txids.as_array().chain_err(|| "non-array result")? {
|
||||
result.insert(parse_hash(&value).chain_err(|| "invalid txid")?);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn broadcast(&self, tx: &Transaction) -> Result<Txid> {
|
||||
pub fn getmempoolentry(&self, txid: &Sha256dHash) -> Result<MempoolEntry> {
|
||||
let entry = self.request("getmempoolentry", json!([txid.to_hex()]))?;
|
||||
let fee = (entry
|
||||
.get("fee")
|
||||
.chain_err(|| "missing fee")?
|
||||
.as_f64()
|
||||
.chain_err(|| "non-float fee")?
|
||||
* 100_000_000f64) as u64;
|
||||
let vsize = entry
|
||||
.get("size")
|
||||
.chain_err(|| "missing size")?
|
||||
.as_u64()
|
||||
.chain_err(|| "non-integer size")? as u32;
|
||||
Ok(MempoolEntry::new(fee, vsize))
|
||||
}
|
||||
|
||||
pub fn broadcast(&self, tx: &Transaction) -> Result<Sha256dHash> {
|
||||
self.broadcast_raw(&hex::encode(serialize(tx)))
|
||||
}
|
||||
|
||||
pub fn broadcast_raw(&self, txhex: &str) -> Result<Txid> {
|
||||
pub fn broadcast_raw(&self, txhex: &String) -> Result<Sha256dHash> {
|
||||
let txid = self.request("sendrawtransaction", json!([txhex]))?;
|
||||
txid.as_str()
|
||||
.chain_err(|| "non-string txid")?
|
||||
.parse::<Txid>()
|
||||
.map_err(|e| format!("failed to parse txid: {:?}", e).into())
|
||||
Ok(
|
||||
Sha256dHash::from_hex(txid.as_str().chain_err(|| "non-string txid")?)
|
||||
.chain_err(|| "failed to parse txid")?,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn test_mempool_accept(
|
||||
&self,
|
||||
txhex: Vec<String>,
|
||||
maxfeerate: Option<f64>,
|
||||
) -> Result<Vec<MempoolAcceptResult>> {
|
||||
let params = match maxfeerate {
|
||||
Some(rate) => json!([txhex, format!("{:.8}", rate)]),
|
||||
None => json!([txhex]),
|
||||
};
|
||||
let result = self.request("testmempoolaccept", params)?;
|
||||
serde_json::from_value::<Vec<MempoolAcceptResult>>(result)
|
||||
.chain_err(|| "invalid testmempoolaccept reply")
|
||||
pub fn estimatesmartfee(&self, conf_target: u16) -> Result<f32> {
|
||||
let feerate = self
|
||||
.request("estimatesmartfee", json!([conf_target]))?
|
||||
.get("feerate")
|
||||
.chain_err(|| "missing feerate")?
|
||||
.as_f64()
|
||||
.chain_err(|| "invalid feerate")?;
|
||||
|
||||
if feerate == -1f64 {
|
||||
bail!("not enough data to estimate");
|
||||
}
|
||||
|
||||
// from BTC/kB to sat/b
|
||||
Ok((feerate * 100_000f64) as f32)
|
||||
}
|
||||
|
||||
pub fn submit_package(
|
||||
&self,
|
||||
txhex: Vec<String>,
|
||||
maxfeerate: Option<f64>,
|
||||
maxburnamount: Option<f64>,
|
||||
) -> Result<SubmitPackageResult> {
|
||||
let params = match (maxfeerate, maxburnamount) {
|
||||
(Some(rate), Some(burn)) => {
|
||||
json!([txhex, format!("{:.8}", rate), format!("{:.8}", burn)])
|
||||
}
|
||||
(Some(rate), None) => json!([txhex, format!("{:.8}", rate)]),
|
||||
(None, Some(burn)) => json!([txhex, null, format!("{:.8}", burn)]),
|
||||
(None, None) => json!([txhex]),
|
||||
};
|
||||
let result = self.request("submitpackage", params)?;
|
||||
serde_json::from_value::<SubmitPackageResult>(result)
|
||||
.chain_err(|| "invalid submitpackage reply")
|
||||
}
|
||||
|
||||
// Get estimated feerates for the provided confirmation targets using a batch RPC request
|
||||
// Missing estimates are logged but do not cause a failure, whatever is available is returned
|
||||
#[allow(clippy::float_cmp)]
|
||||
pub fn estimatesmartfee_batch(&self, conf_targets: &[u16]) -> Result<HashMap<u16, f64>> {
|
||||
let params_list: Vec<Value> = conf_targets.iter().map(|t| json!([t])).collect();
|
||||
|
||||
Ok(self
|
||||
.requests("estimatesmartfee", ¶ms_list)?
|
||||
.iter()
|
||||
.zip(conf_targets)
|
||||
.filter_map(|(reply, target)| {
|
||||
if !reply["errors"].is_null() {
|
||||
warn!(
|
||||
"failed estimating fee for target {}: {:?}",
|
||||
target, reply["errors"]
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
let feerate = reply["feerate"]
|
||||
.as_f64()
|
||||
.unwrap_or_else(|| panic!("invalid estimatesmartfee response: {:?}", reply));
|
||||
|
||||
if feerate == -1f64 {
|
||||
warn!("not enough data to estimate fee for target {}", target);
|
||||
return None;
|
||||
}
|
||||
|
||||
// from BTC/kB to sat/b
|
||||
Some((*target, feerate * 100_000f64))
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn get_all_headers(&self, tip: &BlockHash) -> Result<Vec<BlockHeader>> {
|
||||
let info: Value = self.request("getblockheader", json!([tip.to_string()]))?;
|
||||
fn get_all_headers(&self, tip: &Sha256dHash) -> Result<Vec<BlockHeader>> {
|
||||
let info: Value = self.request("getblockheader", json!([tip.to_hex()]))?;
|
||||
let tip_height = info
|
||||
.get("height")
|
||||
.expect("missing height")
|
||||
.as_u64()
|
||||
.expect("non-numeric height") as usize;
|
||||
let all_heights: Vec<usize> = (0..=tip_height).collect();
|
||||
let all_heights: Vec<usize> = (0..tip_height + 1).collect();
|
||||
let chunk_size = 100_000;
|
||||
let mut result = vec![];
|
||||
let null_hash = Sha256dHash::default();
|
||||
for heights in all_heights.chunks(chunk_size) {
|
||||
trace!("downloading {} block headers", heights.len());
|
||||
let mut headers = self.getblockheaders(heights)?;
|
||||
let mut headers = self.getblockheaders(&heights)?;
|
||||
assert!(headers.len() == heights.len());
|
||||
result.append(&mut headers);
|
||||
}
|
||||
|
||||
let mut blockhash = BlockHash::all_zeros();
|
||||
let mut blockhash = null_hash;
|
||||
for header in &result {
|
||||
assert_eq!(header.prev_blockhash, blockhash);
|
||||
blockhash = header.block_hash();
|
||||
blockhash = header.bitcoin_hash();
|
||||
}
|
||||
assert_eq!(blockhash, *tip);
|
||||
Ok(result)
|
||||
@ -740,10 +608,10 @@ impl Daemon {
|
||||
pub fn get_new_headers(
|
||||
&self,
|
||||
indexed_headers: &HeaderList,
|
||||
bestblockhash: &BlockHash,
|
||||
bestblockhash: &Sha256dHash,
|
||||
) -> Result<Vec<BlockHeader>> {
|
||||
// Iterate back over headers until known blockash is found:
|
||||
if indexed_headers.is_empty() {
|
||||
if indexed_headers.len() == 0 {
|
||||
debug!("downloading all block headers up to {}", bestblockhash);
|
||||
return self.get_all_headers(bestblockhash);
|
||||
}
|
||||
@ -753,7 +621,7 @@ impl Daemon {
|
||||
bestblockhash,
|
||||
);
|
||||
let mut new_headers = vec![];
|
||||
let null_hash = BlockHash::all_zeros();
|
||||
let null_hash = Sha256dHash::default();
|
||||
let mut blockhash = *bestblockhash;
|
||||
while blockhash != null_hash {
|
||||
if indexed_headers.header_by_blockhash(&blockhash).is_some() {
|
||||
@ -769,11 +637,4 @@ impl Daemon {
|
||||
new_headers.reverse(); // so the tip is the last vector entry
|
||||
Ok(new_headers)
|
||||
}
|
||||
|
||||
pub fn get_relayfee(&self) -> Result<f64> {
|
||||
let relayfee = self.getnetworkinfo()?.relayfee;
|
||||
|
||||
// from BTC/kB to sat/b
|
||||
Ok(relayfee * 100_000f64)
|
||||
}
|
||||
}
|
||||
|
||||
630
src/electrum.rs
Normal file
630
src/electrum.rs
Normal file
@ -0,0 +1,630 @@
|
||||
use bitcoin::consensus::encode::serialize;
|
||||
use bitcoin_hashes::hex::{FromHex, ToHex};
|
||||
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
|
||||
use crypto::digest::Digest;
|
||||
use crypto::sha2::Sha256;
|
||||
use error_chain::ChainedError;
|
||||
use hex;
|
||||
use serde_json::{from_str, Value};
|
||||
use std::collections::HashMap;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream};
|
||||
use std::sync::mpsc::{Sender, SyncSender, TrySendError};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
|
||||
use crate::errors::*;
|
||||
use crate::metrics::{Gauge, HistogramOpts, HistogramVec, MetricOpts, Metrics};
|
||||
use crate::new_index::Query;
|
||||
use crate::util::{
|
||||
full_hash, get_header_merkle_proof, get_id_from_pos, get_tx_merkle_proof, spawn_thread,
|
||||
BlockId, Channel, FullHash, HeaderEntry, SyncChannel,
|
||||
};
|
||||
|
||||
// TODO: Sha256dHash should be a generic hash-container (since script hash is single SHA256)
|
||||
fn hash_from_value(val: Option<&Value>) -> Result<Sha256dHash> {
|
||||
let script_hash = val.chain_err(|| "missing hash")?;
|
||||
let script_hash = script_hash.as_str().chain_err(|| "non-string hash")?;
|
||||
let script_hash = Sha256dHash::from_hex(script_hash).chain_err(|| "non-hex hash")?;
|
||||
Ok(script_hash)
|
||||
}
|
||||
|
||||
fn usize_from_value(val: Option<&Value>, name: &str) -> Result<usize> {
|
||||
let val = val.chain_err(|| format!("missing {}", name))?;
|
||||
let val = val.as_u64().chain_err(|| format!("non-integer {}", name))?;
|
||||
Ok(val as usize)
|
||||
}
|
||||
|
||||
fn usize_from_value_or(val: Option<&Value>, name: &str, default: usize) -> Result<usize> {
|
||||
if val.is_none() {
|
||||
return Ok(default);
|
||||
}
|
||||
usize_from_value(val, name)
|
||||
}
|
||||
|
||||
fn bool_from_value(val: Option<&Value>, name: &str) -> Result<bool> {
|
||||
let val = val.chain_err(|| format!("missing {}", name))?;
|
||||
let val = val.as_bool().chain_err(|| format!("not a bool {}", name))?;
|
||||
Ok(val)
|
||||
}
|
||||
|
||||
fn bool_from_value_or(val: Option<&Value>, name: &str, default: bool) -> Result<bool> {
|
||||
if val.is_none() {
|
||||
return Ok(default);
|
||||
}
|
||||
bool_from_value(val, name)
|
||||
}
|
||||
|
||||
// FIXME: implement caching and delta updates
|
||||
// FIXME: ensure stable ordering
|
||||
fn get_status_hash(txs: Vec<(Sha256dHash, Option<BlockId>)>) -> Option<FullHash> {
|
||||
if txs.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let mut hash = FullHash::default();
|
||||
let mut sha2 = Sha256::new();
|
||||
for (txid, blockid) in txs {
|
||||
// TODO: use height of 0 to indicate a mempool tx with confirmed inputs
|
||||
let part = format!(
|
||||
"{}:{}:",
|
||||
txid.to_hex(),
|
||||
blockid.map_or(-1, |b| b.height as isize)
|
||||
);
|
||||
sha2.input(part.as_bytes());
|
||||
}
|
||||
sha2.result(&mut hash);
|
||||
Some(hash)
|
||||
}
|
||||
}
|
||||
|
||||
struct Connection {
|
||||
query: Arc<Query>,
|
||||
last_header_entry: Option<HeaderEntry>,
|
||||
status_hashes: HashMap<Sha256dHash, Value>, // ScriptHash -> StatusHash
|
||||
stream: TcpStream,
|
||||
addr: SocketAddr,
|
||||
chan: SyncChannel<Message>,
|
||||
stats: Arc<Stats>,
|
||||
}
|
||||
|
||||
impl Connection {
|
||||
pub fn new(
|
||||
query: Arc<Query>,
|
||||
stream: TcpStream,
|
||||
addr: SocketAddr,
|
||||
stats: Arc<Stats>,
|
||||
) -> Connection {
|
||||
Connection {
|
||||
query,
|
||||
last_header_entry: None, // disable header subscription for now
|
||||
status_hashes: HashMap::new(),
|
||||
stream,
|
||||
addr,
|
||||
chan: SyncChannel::new(10),
|
||||
stats,
|
||||
}
|
||||
}
|
||||
|
||||
fn blockchain_headers_subscribe(&mut self) -> Result<Value> {
|
||||
let entry = self.query.chain().best_header();
|
||||
let hex_header = hex::encode(serialize(entry.header()));
|
||||
let result = json!({"hex": hex_header, "height": entry.height()});
|
||||
self.last_header_entry = Some(entry);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn server_version(&self) -> Result<Value> {
|
||||
// TODO dynamic version
|
||||
Ok(json!(["electrs-esplora", "1.4"]))
|
||||
}
|
||||
|
||||
fn server_banner(&self) -> Result<Value> {
|
||||
// TODO dynamic banner
|
||||
Ok(json!("Welcome to electrs-esplora"))
|
||||
}
|
||||
|
||||
fn server_donation_address(&self) -> Result<Value> {
|
||||
Ok(Value::Null)
|
||||
}
|
||||
|
||||
fn server_peers_subscribe(&self) -> Result<Value> {
|
||||
Ok(json!([]))
|
||||
}
|
||||
|
||||
fn mempool_get_fee_histogram(&self) -> Result<Value> {
|
||||
Ok(json!(&self.query.mempool().backlog_stats().fee_histogram))
|
||||
}
|
||||
|
||||
fn blockchain_block_header(&self, params: &[Value]) -> Result<Value> {
|
||||
let height = usize_from_value(params.get(0), "height")?;
|
||||
let cp_height = usize_from_value_or(params.get(1), "cp_height", 0)?;
|
||||
|
||||
let raw_header_hex: String = self
|
||||
.query
|
||||
.chain()
|
||||
.header_by_height(height)
|
||||
.map(|entry| hex::encode(&serialize(entry.header())))
|
||||
.chain_err(|| "missing header")?;
|
||||
|
||||
if cp_height == 0 {
|
||||
return Ok(json!(raw_header_hex));
|
||||
}
|
||||
let (branch, root) = get_header_merkle_proof(self.query.chain(), height, cp_height)?;
|
||||
|
||||
let branch_vec: Vec<String> = branch.into_iter().map(|b| b.to_hex()).collect();
|
||||
|
||||
return Ok(json!({
|
||||
"header": raw_header_hex,
|
||||
"root": root.to_hex(),
|
||||
"branch": branch_vec
|
||||
}));
|
||||
}
|
||||
|
||||
fn blockchain_block_headers(&self, params: &[Value]) -> Result<Value> {
|
||||
let start_height = usize_from_value(params.get(0), "start_height")?;
|
||||
let count = usize_from_value(params.get(1), "count")?;
|
||||
let cp_height = usize_from_value_or(params.get(2), "cp_height", 0)?;
|
||||
let heights: Vec<usize> = (start_height..(start_height + count)).collect();
|
||||
let headers: Vec<String> = heights
|
||||
.into_iter()
|
||||
.map(|height| {
|
||||
self.query
|
||||
.chain()
|
||||
.header_by_height(height)
|
||||
.map(|entry| hex::encode(&serialize(entry.header())))
|
||||
})
|
||||
.collect::<Option<Vec<String>>>()
|
||||
.chain_err(|| "requested block headers not found")?;
|
||||
|
||||
if count == 0 || cp_height == 0 {
|
||||
return Ok(json!({
|
||||
"count": headers.len(),
|
||||
"hex": headers.join(""),
|
||||
"max": 2016,
|
||||
}));
|
||||
}
|
||||
|
||||
let (branch, root) =
|
||||
get_header_merkle_proof(self.query.chain(), start_height + (count - 1), cp_height)?;
|
||||
|
||||
let branch_vec: Vec<String> = branch.into_iter().map(|b| b.to_hex()).collect();
|
||||
|
||||
Ok(json!({
|
||||
"count": headers.len(),
|
||||
"hex": headers.join(""),
|
||||
"max": 2016,
|
||||
"root": root.to_hex(),
|
||||
"branch" : branch_vec
|
||||
}))
|
||||
}
|
||||
|
||||
fn blockchain_estimatefee(&self, params: &[Value]) -> Result<Value> {
|
||||
let conf_target = usize_from_value(params.get(0), "blocks_count")?;
|
||||
let fee_rate = self
|
||||
.query
|
||||
.estimate_fee(conf_target as u16)
|
||||
.chain_err(|| format!("cannot estimate fee for {} blocks", conf_target))?;
|
||||
// convert from sat/b to BTC/kB, as expected by Electrum clients
|
||||
Ok(json!(fee_rate / 100_000f32))
|
||||
}
|
||||
|
||||
fn blockchain_relayfee(&self) -> Result<Value> {
|
||||
// TODO
|
||||
Ok(json!(0.0)) // allow sending transactions with any fee.
|
||||
}
|
||||
|
||||
fn blockchain_scripthash_subscribe(&mut self, params: &[Value]) -> Result<Value> {
|
||||
let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
|
||||
|
||||
let history_txids = self.query.history_txids(&script_hash[..]);
|
||||
let status_hash = get_status_hash(history_txids)
|
||||
.map_or(Value::Null, |h| json!(hex::encode(full_hash(&h[..]))));
|
||||
|
||||
self.status_hashes.insert(script_hash, status_hash.clone());
|
||||
Ok(status_hash)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
fn blockchain_scripthash_get_balance(&self, params: &[Value]) -> Result<Value> {
|
||||
let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
|
||||
let (chain_stats, mempool_stats) = self.query.stats(&script_hash[..]);
|
||||
|
||||
Ok(
|
||||
json!({ "confirmed": chain_stats.funded_txo_sum - chain_stats.spent_txo_sum, "unconfirmed": mempool_stats.funded_txo_sum - mempool_stats.spent_txo_sum}),
|
||||
)
|
||||
}
|
||||
|
||||
fn blockchain_scripthash_get_history(&self, params: &[Value]) -> Result<Value> {
|
||||
let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
|
||||
let history_txids = self.query.history_txids(&script_hash[..]);
|
||||
Ok(json!(Value::Array(
|
||||
history_txids
|
||||
.into_iter()
|
||||
.map(|(txid, blockid)| json!({"height": blockid.map_or(0, |b| b.height), "tx_hash": txid.to_hex()}))
|
||||
.collect()
|
||||
)))
|
||||
}
|
||||
|
||||
fn blockchain_scripthash_listunspent(&self, params: &[Value]) -> Result<Value> {
|
||||
let script_hash = hash_from_value(params.get(0)).chain_err(|| "bad script_hash")?;
|
||||
let utxos = self.query.utxo(&script_hash[..]);
|
||||
Ok(json!(Value::Array(
|
||||
utxos
|
||||
.into_iter()
|
||||
.map(|utxo| json!({
|
||||
"height": utxo.confirmed.map_or(0, |b| b.height),
|
||||
"tx_pos": utxo.vout,
|
||||
"tx_hash": utxo.txid.to_hex(),
|
||||
"value": utxo.value,
|
||||
}))
|
||||
.collect()
|
||||
)))
|
||||
}
|
||||
|
||||
fn blockchain_transaction_broadcast(&self, params: &[Value]) -> Result<Value> {
|
||||
let tx = params.get(0).chain_err(|| "missing tx")?;
|
||||
let tx = tx.as_str().chain_err(|| "non-string tx")?.to_string();
|
||||
let txid = self.query.broadcast_raw(&tx)?;
|
||||
if let Err(e) = self.chan.sender().try_send(Message::PeriodicUpdate) {
|
||||
warn!("failed to issue PeriodicUpdate after broadcast: {}", e);
|
||||
}
|
||||
Ok(json!(txid.to_hex()))
|
||||
}
|
||||
|
||||
fn blockchain_transaction_get(&self, params: &[Value]) -> Result<Value> {
|
||||
let tx_hash = hash_from_value(params.get(0)).chain_err(|| "bad tx_hash")?;
|
||||
let verbose = match params.get(1) {
|
||||
Some(value) => value.as_bool().chain_err(|| "non-bool verbose value")?,
|
||||
None => false,
|
||||
};
|
||||
|
||||
// FIXME: implement verbose support
|
||||
if verbose {
|
||||
bail!("verbose transactions are currently unsupported");
|
||||
}
|
||||
|
||||
let tx = self
|
||||
.query
|
||||
.lookup_raw_txn(&tx_hash)
|
||||
.chain_err(|| "missing transaction")?;
|
||||
Ok(json!(hex::encode(tx)))
|
||||
}
|
||||
|
||||
fn blockchain_transaction_get_merkle(&self, params: &[Value]) -> Result<Value> {
|
||||
let txid = hash_from_value(params.get(0)).chain_err(|| "bad tx_hash")?;
|
||||
let height = usize_from_value(params.get(1), "height")?;
|
||||
let blockid = self
|
||||
.query
|
||||
.chain()
|
||||
.tx_confirming_block(&txid)
|
||||
.ok_or_else(|| "tx not found or is unconfirmed")?;
|
||||
if blockid.height != height {
|
||||
bail!("invalid confirmation height provided");
|
||||
}
|
||||
let (merkle, pos) = get_tx_merkle_proof(self.query.chain(), &txid, &blockid.hash)
|
||||
.chain_err(|| "cannot create merkle proof")?;
|
||||
let merkle: Vec<String> = merkle.into_iter().map(|txid| txid.to_hex()).collect();
|
||||
Ok(json!({
|
||||
"block_height": blockid.height,
|
||||
"merkle": merkle,
|
||||
"pos": pos}))
|
||||
}
|
||||
|
||||
fn blockchain_transaction_id_from_pos(&self, params: &[Value]) -> Result<Value> {
|
||||
let height = usize_from_value(params.get(0), "height")?;
|
||||
let tx_pos = usize_from_value(params.get(1), "tx_pos")?;
|
||||
let want_merkle = bool_from_value_or(params.get(2), "merkle", false)?;
|
||||
|
||||
let (txid, merkle) = get_id_from_pos(self.query.chain(), height, tx_pos, want_merkle)?;
|
||||
|
||||
if !want_merkle {
|
||||
return Ok(json!(txid.to_hex()));
|
||||
}
|
||||
|
||||
let merkle_vec: Vec<String> = merkle.into_iter().map(|entry| entry.to_hex()).collect();
|
||||
|
||||
Ok(json!({
|
||||
"tx_hash" : txid.to_hex(),
|
||||
"merkle" : merkle_vec}))
|
||||
}
|
||||
|
||||
fn handle_command(&mut self, method: &str, params: &[Value], id: &Value) -> Result<Value> {
|
||||
let timer = self
|
||||
.stats
|
||||
.latency
|
||||
.with_label_values(&[method])
|
||||
.start_timer();
|
||||
let result = match method {
|
||||
"blockchain.block.header" => self.blockchain_block_header(¶ms),
|
||||
"blockchain.block.headers" => self.blockchain_block_headers(¶ms),
|
||||
"blockchain.estimatefee" => self.blockchain_estimatefee(¶ms),
|
||||
"blockchain.headers.subscribe" => self.blockchain_headers_subscribe(),
|
||||
"blockchain.relayfee" => self.blockchain_relayfee(),
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
"blockchain.scripthash.get_balance" => self.blockchain_scripthash_get_balance(¶ms),
|
||||
"blockchain.scripthash.get_history" => self.blockchain_scripthash_get_history(¶ms),
|
||||
"blockchain.scripthash.listunspent" => self.blockchain_scripthash_listunspent(¶ms),
|
||||
"blockchain.scripthash.subscribe" => self.blockchain_scripthash_subscribe(¶ms),
|
||||
"blockchain.transaction.broadcast" => self.blockchain_transaction_broadcast(¶ms),
|
||||
"blockchain.transaction.get" => self.blockchain_transaction_get(¶ms),
|
||||
"blockchain.transaction.get_merkle" => self.blockchain_transaction_get_merkle(¶ms),
|
||||
"blockchain.transaction.id_from_pos" => {
|
||||
self.blockchain_transaction_id_from_pos(¶ms)
|
||||
}
|
||||
"mempool.get_fee_histogram" => self.mempool_get_fee_histogram(),
|
||||
"server.banner" => self.server_banner(),
|
||||
"server.donation_address" => self.server_donation_address(),
|
||||
"server.peers.subscribe" => self.server_peers_subscribe(),
|
||||
"server.ping" => Ok(Value::Null),
|
||||
"server.version" => self.server_version(),
|
||||
&_ => bail!("unknown method {} {:?}", method, params),
|
||||
};
|
||||
timer.observe_duration();
|
||||
// TODO: return application errors should be sent to the client
|
||||
Ok(match result {
|
||||
Ok(result) => json!({"jsonrpc": "2.0", "id": id, "result": result}),
|
||||
Err(e) => {
|
||||
warn!(
|
||||
"rpc #{} {} {:?} failed: {}",
|
||||
id,
|
||||
method,
|
||||
params,
|
||||
e.display_chain()
|
||||
);
|
||||
json!({"jsonrpc": "2.0", "id": id, "error": format!("{}", e)})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn update_subscriptions(&mut self) -> Result<Vec<Value>> {
|
||||
let timer = self
|
||||
.stats
|
||||
.latency
|
||||
.with_label_values(&["periodic_update"])
|
||||
.start_timer();
|
||||
let mut result = vec![];
|
||||
if let Some(ref mut last_entry) = self.last_header_entry {
|
||||
let entry = self.query.chain().best_header();
|
||||
if *last_entry != entry {
|
||||
*last_entry = entry;
|
||||
let hex_header = hex::encode(serialize(last_entry.header()));
|
||||
let header = json!({"hex": hex_header, "height": last_entry.height()});
|
||||
result.push(json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": "blockchain.headers.subscribe",
|
||||
"params": [header]}));
|
||||
}
|
||||
}
|
||||
for (script_hash, status_hash) in self.status_hashes.iter_mut() {
|
||||
let history_txids = self.query.history_txids(&script_hash[..]);
|
||||
let new_status_hash = get_status_hash(history_txids)
|
||||
.map_or(Value::Null, |h| json!(hex::encode(full_hash(&h[..]))));
|
||||
if new_status_hash == *status_hash {
|
||||
continue;
|
||||
}
|
||||
result.push(json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": "blockchain.scripthash.subscribe",
|
||||
"params": [script_hash.to_hex(), new_status_hash]}));
|
||||
*status_hash = new_status_hash;
|
||||
}
|
||||
timer.observe_duration();
|
||||
self.stats
|
||||
.subscriptions
|
||||
.set(self.status_hashes.len() as i64);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn send_values(&mut self, values: &[Value]) -> Result<()> {
|
||||
for value in values {
|
||||
let line = value.to_string() + "\n";
|
||||
self.stream
|
||||
.write_all(line.as_bytes())
|
||||
.chain_err(|| format!("failed to send {}", value))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn handle_replies(&mut self) -> Result<()> {
|
||||
let empty_params = json!([]);
|
||||
loop {
|
||||
let msg = self.chan.receiver().recv().chain_err(|| "channel closed")?;
|
||||
trace!("RPC {:?}", msg);
|
||||
match msg {
|
||||
Message::Request(line) => {
|
||||
let cmd: Value = from_str(&line).chain_err(|| "invalid JSON format")?;
|
||||
let reply = match (
|
||||
cmd.get("method"),
|
||||
cmd.get("params").unwrap_or_else(|| &empty_params),
|
||||
cmd.get("id"),
|
||||
) {
|
||||
(
|
||||
Some(&Value::String(ref method)),
|
||||
&Value::Array(ref params),
|
||||
Some(ref id),
|
||||
) => self.handle_command(method, params, id)?,
|
||||
_ => bail!("invalid command: {}", cmd),
|
||||
};
|
||||
self.send_values(&[reply])?
|
||||
}
|
||||
Message::PeriodicUpdate => {
|
||||
let values = self
|
||||
.update_subscriptions()
|
||||
.chain_err(|| "failed to update subscriptions")?;
|
||||
self.send_values(&values)?
|
||||
}
|
||||
Message::Done => return Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_requests(mut reader: BufReader<TcpStream>, tx: SyncSender<Message>) -> Result<()> {
|
||||
loop {
|
||||
let mut line = Vec::<u8>::new();
|
||||
reader
|
||||
.read_until(b'\n', &mut line)
|
||||
.chain_err(|| "failed to read a request")?;
|
||||
if line.is_empty() {
|
||||
tx.send(Message::Done).chain_err(|| "channel closed")?;
|
||||
return Ok(());
|
||||
} else {
|
||||
if line.starts_with(&[22, 3, 1]) {
|
||||
// (very) naive SSL handshake detection
|
||||
let _ = tx.send(Message::Done);
|
||||
bail!("invalid request - maybe SSL-encrypted data?: {:?}", line)
|
||||
}
|
||||
match String::from_utf8(line) {
|
||||
Ok(req) => tx
|
||||
.send(Message::Request(req))
|
||||
.chain_err(|| "channel closed")?,
|
||||
Err(err) => {
|
||||
let _ = tx.send(Message::Done);
|
||||
bail!("invalid UTF8: {}", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run(mut self) {
|
||||
let reader = BufReader::new(self.stream.try_clone().expect("failed to clone TcpStream"));
|
||||
let tx = self.chan.sender();
|
||||
let child = spawn_thread("reader", || Connection::handle_requests(reader, tx));
|
||||
if let Err(e) = self.handle_replies() {
|
||||
error!(
|
||||
"[{}] connection handling failed: {}",
|
||||
self.addr,
|
||||
e.display_chain().to_string()
|
||||
);
|
||||
}
|
||||
debug!("[{}] shutting down connection", self.addr);
|
||||
let _ = self.stream.shutdown(Shutdown::Both);
|
||||
if let Err(err) = child.join().expect("receiver panicked") {
|
||||
error!("[{}] receiver failed: {}", self.addr, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Message {
|
||||
Request(String),
|
||||
PeriodicUpdate,
|
||||
Done,
|
||||
}
|
||||
|
||||
pub enum Notification {
|
||||
Periodic,
|
||||
Exit,
|
||||
}
|
||||
|
||||
pub struct RPC {
|
||||
notification: Sender<Notification>,
|
||||
server: Option<thread::JoinHandle<()>>, // so we can join the server while dropping this ojbect
|
||||
}
|
||||
|
||||
struct Stats {
|
||||
latency: HistogramVec,
|
||||
subscriptions: Gauge,
|
||||
}
|
||||
|
||||
impl RPC {
|
||||
fn start_notifier(
|
||||
notification: Channel<Notification>,
|
||||
senders: Arc<Mutex<Vec<SyncSender<Message>>>>,
|
||||
acceptor: Sender<Option<(TcpStream, SocketAddr)>>,
|
||||
) {
|
||||
spawn_thread("notification", move || {
|
||||
for msg in notification.receiver().iter() {
|
||||
let mut senders = senders.lock().unwrap();
|
||||
match msg {
|
||||
Notification::Periodic => {
|
||||
for sender in senders.split_off(0) {
|
||||
if let Err(TrySendError::Disconnected(_)) =
|
||||
sender.try_send(Message::PeriodicUpdate)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
senders.push(sender);
|
||||
}
|
||||
}
|
||||
Notification::Exit => acceptor.send(None).unwrap(),
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn start_acceptor(addr: SocketAddr) -> Channel<Option<(TcpStream, SocketAddr)>> {
|
||||
let chan = Channel::new();
|
||||
let acceptor = chan.sender();
|
||||
spawn_thread("acceptor", move || {
|
||||
let listener = TcpListener::bind(addr).expect(&format!("bind({}) failed", addr));
|
||||
info!("Electrum RPC server running on {}", addr);
|
||||
loop {
|
||||
let (stream, addr) = listener.accept().expect("accept failed");
|
||||
stream
|
||||
.set_nonblocking(false)
|
||||
.expect("failed to set connection as blocking");
|
||||
acceptor.send(Some((stream, addr))).expect("send failed");
|
||||
}
|
||||
});
|
||||
chan
|
||||
}
|
||||
|
||||
pub fn start(addr: SocketAddr, query: Arc<Query>, metrics: &Metrics) -> RPC {
|
||||
let stats = Arc::new(Stats {
|
||||
latency: metrics.histogram_vec(
|
||||
HistogramOpts::new("electrum_rpc", "Electrum RPC latency (seconds)"),
|
||||
&["method"],
|
||||
),
|
||||
subscriptions: metrics.gauge(MetricOpts::new(
|
||||
"electrum_subscriptions",
|
||||
"# of Electrum subscriptions",
|
||||
)),
|
||||
});
|
||||
let notification = Channel::new();
|
||||
let handle = RPC {
|
||||
notification: notification.sender(),
|
||||
server: Some(spawn_thread("rpc", move || {
|
||||
let senders = Arc::new(Mutex::new(Vec::<SyncSender<Message>>::new()));
|
||||
let acceptor = RPC::start_acceptor(addr);
|
||||
RPC::start_notifier(notification, senders.clone(), acceptor.sender());
|
||||
let mut children = vec![];
|
||||
while let Some((stream, addr)) = acceptor.receiver().recv().unwrap() {
|
||||
let query = query.clone();
|
||||
let senders = senders.clone();
|
||||
let stats = stats.clone();
|
||||
children.push(spawn_thread("peer", move || {
|
||||
info!("[{}] connected peer", addr);
|
||||
let conn = Connection::new(query, stream, addr, stats);
|
||||
senders.lock().unwrap().push(conn.chan.sender());
|
||||
conn.run();
|
||||
info!("[{}] disconnected peer", addr);
|
||||
}));
|
||||
}
|
||||
trace!("closing {} RPC connections", senders.lock().unwrap().len());
|
||||
for sender in senders.lock().unwrap().iter() {
|
||||
let _ = sender.send(Message::Done);
|
||||
}
|
||||
trace!("waiting for {} RPC handling threads", children.len());
|
||||
for child in children {
|
||||
let _ = child.join();
|
||||
}
|
||||
trace!("RPC connections are closed");
|
||||
})),
|
||||
};
|
||||
handle
|
||||
}
|
||||
|
||||
pub fn notify(&self) {
|
||||
self.notification.send(Notification::Periodic).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RPC {
|
||||
fn drop(&mut self) {
|
||||
trace!("stop accepting new RPCs");
|
||||
self.notification.send(Notification::Exit).unwrap();
|
||||
self.server.take().map(|t| t.join().unwrap());
|
||||
trace!("RPC server is stopped");
|
||||
}
|
||||
}
|
||||
@ -1,41 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
use std::convert::TryFrom;
|
||||
|
||||
use bitcoin::hashes::sha256d;
|
||||
use bitcoin::hashes::Hash;
|
||||
pub use electrum_client::client::Client;
|
||||
pub use electrum_client::ServerFeaturesRes;
|
||||
|
||||
use crate::chain::BlockHash;
|
||||
use crate::electrum::ServerFeatures;
|
||||
use crate::errors::{Error, ResultExt};
|
||||
|
||||
// Convert from electrum-client's server features struct to ours. We're using a different struct because
|
||||
// the electrum-client's one doesn't support the "hosts" key.
|
||||
impl TryFrom<ServerFeaturesRes> for ServerFeatures {
|
||||
type Error = Error;
|
||||
fn try_from(mut features: ServerFeaturesRes) -> Result<Self, Self::Error> {
|
||||
features.genesis_hash.reverse();
|
||||
|
||||
Ok(ServerFeatures {
|
||||
// electrum-client doesn't retain the hosts map data, but we already have it from the add_peer request
|
||||
hosts: HashMap::new(),
|
||||
genesis_hash: BlockHash::from_raw_hash(sha256d::Hash::from_byte_array(
|
||||
features.genesis_hash,
|
||||
)),
|
||||
server_version: features.server_version,
|
||||
protocol_min: features
|
||||
.protocol_min
|
||||
.parse()
|
||||
.chain_err(|| "invalid protocol_min")?,
|
||||
protocol_max: features
|
||||
.protocol_max
|
||||
.parse()
|
||||
.chain_err(|| "invalid protocol_max")?,
|
||||
pruning: features.pruning.map(|pruning| pruning as usize),
|
||||
hash_function: features
|
||||
.hash_function
|
||||
.chain_err(|| "missing hash_function")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,603 +0,0 @@
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::{hash_map::Entry, BinaryHeap, HashMap, HashSet};
|
||||
use std::convert::TryInto;
|
||||
use std::fmt;
|
||||
use std::net::{IpAddr, SocketAddr, ToSocketAddrs};
|
||||
use std::str::FromStr;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use electrum_client::ElectrumApi;
|
||||
|
||||
use crate::chain::Network;
|
||||
use crate::electrum::{Client, Hostname, Port, ProtocolVersion, ServerFeatures};
|
||||
use crate::errors::{Result, ResultExt};
|
||||
use crate::util::spawn_thread;
|
||||
|
||||
mod default_servers;
|
||||
use default_servers::add_default_servers;
|
||||
|
||||
const HEALTH_CHECK_FREQ: Duration = Duration::from_secs(3600); // check servers every hour
|
||||
const JOB_INTERVAL: Duration = Duration::from_secs(1); // run one health check job every second
|
||||
const MAX_CONSECUTIVE_FAILURES: usize = 24; // drop servers after 24 consecutive failing attempts (~24 hours) (~24 hours)
|
||||
const MAX_QUEUE_SIZE: usize = 500; // refuse accepting new servers if we have that many health check jobs
|
||||
const MAX_SERVERS_PER_REQUEST: usize = 3; // maximum number of server hosts added per server.add_peer call
|
||||
const MAX_SERVICES_PER_REQUEST: usize = 6; // maximum number of services added per server.add_peer call
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct DiscoveryManager {
|
||||
/// A queue of scheduled health check jobs, including for healthy, unhealthy and untested servers
|
||||
queue: RwLock<BinaryHeap<HealthCheck>>,
|
||||
|
||||
/// A list of servers that were found to be healthy on their last health check
|
||||
healthy: RwLock<HashMap<ServerAddr, Server>>,
|
||||
|
||||
/// Used to test for protocol version compatibility
|
||||
our_version: ProtocolVersion,
|
||||
|
||||
/// So that we don't list ourselves
|
||||
our_addrs: HashSet<ServerAddr>,
|
||||
|
||||
/// For advertising ourself to other servers
|
||||
our_features: ServerFeatures,
|
||||
|
||||
/// Whether we should announce ourselves to the servers we're connecting to
|
||||
announce: bool,
|
||||
|
||||
/// Optional, will not support onion hosts without this
|
||||
tor_proxy: Option<SocketAddr>,
|
||||
}
|
||||
|
||||
/// A Server corresponds to a single IP address or onion hostname, with one or more services
|
||||
/// exposed on different ports.
|
||||
#[derive(Debug)]
|
||||
struct Server {
|
||||
services: HashSet<Service>,
|
||||
hostname: Hostname,
|
||||
features: ServerFeatures,
|
||||
// the `ServerAddr` isn't kept here directly, but is also available next to `Server` as the key for
|
||||
// the `healthy` field on `DiscoveryManager`
|
||||
}
|
||||
|
||||
#[derive(Eq, PartialEq, Hash, Clone, Debug)]
|
||||
enum ServerAddr {
|
||||
Clearnet(IpAddr),
|
||||
Onion(Hostname),
|
||||
}
|
||||
|
||||
#[derive(Eq, PartialEq, Hash, Copy, Clone, Debug)]
|
||||
pub enum Service {
|
||||
Tcp(Port),
|
||||
Ssl(Port),
|
||||
// unimplemented: Ws and Wss
|
||||
}
|
||||
|
||||
/// A queued health check job, one per service/port (and not per server)
|
||||
#[derive(Eq, Debug)]
|
||||
struct HealthCheck {
|
||||
addr: ServerAddr,
|
||||
hostname: Hostname,
|
||||
service: Service,
|
||||
is_default: bool,
|
||||
#[allow(dead_code)]
|
||||
added_by: Option<IpAddr>,
|
||||
last_check: Option<Instant>,
|
||||
last_healthy: Option<Instant>,
|
||||
consecutive_failures: usize,
|
||||
}
|
||||
|
||||
/// The server entry format returned from server.peers.subscribe
|
||||
#[derive(Serialize)]
|
||||
pub struct ServerEntry(ServerAddr, Hostname, Vec<String>);
|
||||
|
||||
impl DiscoveryManager {
|
||||
pub fn new(
|
||||
our_network: Network,
|
||||
our_features: ServerFeatures,
|
||||
our_version: ProtocolVersion,
|
||||
announce: bool,
|
||||
tor_proxy: Option<SocketAddr>,
|
||||
) -> Self {
|
||||
let our_addrs = our_features
|
||||
.hosts
|
||||
.keys()
|
||||
.filter_map(|hostname| {
|
||||
ServerAddr::resolve(hostname)
|
||||
.map_err(|e| warn!("failed resolving own hostname {}: {:?}", hostname, e))
|
||||
.ok()
|
||||
})
|
||||
.collect();
|
||||
let discovery = Self {
|
||||
our_addrs,
|
||||
our_version,
|
||||
our_features,
|
||||
announce,
|
||||
tor_proxy,
|
||||
healthy: Default::default(),
|
||||
queue: Default::default(),
|
||||
};
|
||||
add_default_servers(&discovery, our_network);
|
||||
discovery
|
||||
}
|
||||
|
||||
/// Add a server requested via `server.add_peer`
|
||||
pub fn add_server_request(&self, added_by: IpAddr, features: ServerFeatures) -> Result<()> {
|
||||
self.verify_compatibility(&features)?;
|
||||
|
||||
let mut queue = self.queue.write().unwrap();
|
||||
ensure!(queue.len() < MAX_QUEUE_SIZE, "queue size exceeded");
|
||||
|
||||
// TODO optimize
|
||||
let mut existing_services: HashMap<ServerAddr, HashSet<Service>> = HashMap::new();
|
||||
for job in queue.iter() {
|
||||
existing_services
|
||||
.entry(job.addr.clone())
|
||||
.or_default()
|
||||
.insert(job.service);
|
||||
}
|
||||
|
||||
// collect HealthChecks for candidate services
|
||||
let jobs = features
|
||||
.hosts
|
||||
.iter()
|
||||
.take(MAX_SERVERS_PER_REQUEST)
|
||||
.filter_map(|(hostname, ports)| {
|
||||
let hostname = hostname.to_lowercase();
|
||||
|
||||
if hostname.len() > 100 {
|
||||
warn!("skipping invalid hostname");
|
||||
return None;
|
||||
}
|
||||
let addr = match ServerAddr::resolve(&hostname) {
|
||||
Ok(addr) => addr,
|
||||
Err(e) => {
|
||||
warn!("failed resolving {}: {:?}", hostname, e);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if !is_remote_addr(&addr) || self.our_addrs.contains(&addr) {
|
||||
warn!("skipping own or non-remote server addr");
|
||||
return None;
|
||||
}
|
||||
// ensure the server address matches the ip that advertised it to us.
|
||||
// onion hosts are exempt.
|
||||
if let ServerAddr::Clearnet(ip) = addr {
|
||||
if ip != added_by {
|
||||
warn!(
|
||||
"server ip does not match source ip ({}, {} != {})",
|
||||
hostname, ip, added_by
|
||||
);
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Some((addr, hostname, ports))
|
||||
})
|
||||
.flat_map(|(addr, hostname, ports)| {
|
||||
let tcp_service = ports.tcp_port.into_iter().map(Service::Tcp);
|
||||
let ssl_service = ports.ssl_port.into_iter().map(Service::Ssl);
|
||||
let services = tcp_service.chain(ssl_service).collect::<HashSet<Service>>();
|
||||
|
||||
services
|
||||
.into_iter()
|
||||
.filter(|service| {
|
||||
existing_services
|
||||
.get(&addr)
|
||||
.is_none_or(|s| !s.contains(service))
|
||||
})
|
||||
.map(|service| {
|
||||
HealthCheck::new(addr.clone(), hostname.clone(), service, Some(added_by))
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.take(MAX_SERVICES_PER_REQUEST)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
ensure!(
|
||||
queue.len() + jobs.len() <= MAX_QUEUE_SIZE,
|
||||
"queue size exceeded"
|
||||
);
|
||||
|
||||
queue.extend(jobs);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Add a default server. Default servers are exempt from limits and given more leniency
|
||||
/// before being removed due to unavailability.
|
||||
pub fn add_default_server(&self, hostname: Hostname, services: Vec<Service>) -> Result<()> {
|
||||
let addr = ServerAddr::resolve(&hostname)?;
|
||||
let mut queue = self.queue.write().unwrap();
|
||||
queue.extend(
|
||||
services
|
||||
.into_iter()
|
||||
.map(|service| HealthCheck::new(addr.clone(), hostname.clone(), service, None)),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the list of healthy servers formatted for `servers.peers.subscribe`
|
||||
pub fn get_servers(&self) -> Vec<ServerEntry> {
|
||||
// XXX return a random sample instead of everything?
|
||||
self.healthy
|
||||
.read()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|(addr, server)| {
|
||||
ServerEntry(addr.clone(), server.hostname.clone(), server.feature_strs())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn our_features(&self) -> &ServerFeatures {
|
||||
&self.our_features
|
||||
}
|
||||
|
||||
/// Run the next health check in the queue (a single one)
|
||||
fn run_health_check(&self) -> Result<()> {
|
||||
// abort if there are no entries in the queue, or its still too early for the next one up
|
||||
if self.queue.read().unwrap().peek().is_none_or(|next| {
|
||||
next.last_check
|
||||
.is_some_and(|t| t.elapsed() < HEALTH_CHECK_FREQ)
|
||||
}) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut job = self.queue.write().unwrap().pop().unwrap();
|
||||
debug!("processing {:?}", job);
|
||||
|
||||
let was_healthy = job.is_healthy();
|
||||
|
||||
match self.check_server(&job.addr, &job.hostname, job.service) {
|
||||
Ok(features) => {
|
||||
debug!("{} {:?} is available", job.hostname, job.service);
|
||||
|
||||
if !was_healthy {
|
||||
self.save_healthy_service(&job, features);
|
||||
}
|
||||
// XXX update features?
|
||||
|
||||
job.last_check = Some(Instant::now());
|
||||
job.last_healthy = job.last_check;
|
||||
job.consecutive_failures = 0;
|
||||
// schedule the next health check
|
||||
self.queue.write().unwrap().push(job);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("{} {:?} is unavailable: {:?}", job.hostname, job.service, e);
|
||||
|
||||
if was_healthy {
|
||||
// XXX should we assume the server's other services are down too?
|
||||
self.remove_unhealthy_service(&job);
|
||||
}
|
||||
|
||||
job.last_check = Some(Instant::now());
|
||||
job.consecutive_failures += 1;
|
||||
|
||||
if job.should_retry() {
|
||||
self.queue.write().unwrap().push(job);
|
||||
} else {
|
||||
debug!("giving up on {:?}", job);
|
||||
}
|
||||
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Upsert the server/service into the healthy set
|
||||
fn save_healthy_service(&self, job: &HealthCheck, features: ServerFeatures) {
|
||||
let addr = job.addr.clone();
|
||||
let mut healthy = self.healthy.write().unwrap();
|
||||
healthy
|
||||
.entry(addr)
|
||||
.or_insert_with(|| Server::new(job.hostname.clone(), features))
|
||||
.services
|
||||
.insert(job.service);
|
||||
}
|
||||
|
||||
/// Remove the service, and remove the server entirely if it has no other reamining healthy services
|
||||
fn remove_unhealthy_service(&self, job: &HealthCheck) {
|
||||
let addr = job.addr.clone();
|
||||
let mut healthy = self.healthy.write().unwrap();
|
||||
if let Entry::Occupied(mut entry) = healthy.entry(addr) {
|
||||
let server = entry.get_mut();
|
||||
assert!(server.services.remove(&job.service));
|
||||
if server.services.is_empty() {
|
||||
entry.remove_entry();
|
||||
}
|
||||
} else {
|
||||
unreachable!("missing expected server, corrupted state");
|
||||
}
|
||||
}
|
||||
|
||||
fn check_server(
|
||||
&self,
|
||||
addr: &ServerAddr,
|
||||
hostname: &Hostname,
|
||||
service: Service,
|
||||
) -> Result<ServerFeatures> {
|
||||
debug!("checking service {:?} {:?}", addr, service);
|
||||
|
||||
let server_url = match (addr, service) {
|
||||
(ServerAddr::Clearnet(ip), Service::Tcp(port)) => format!("tcp://{}:{}", ip, port),
|
||||
(ServerAddr::Clearnet(_), Service::Ssl(port)) => format!("ssl://{}:{}", hostname, port),
|
||||
(ServerAddr::Onion(onion_host), Service::Tcp(port)) => {
|
||||
format!("tcp://{}:{}", onion_host, port)
|
||||
}
|
||||
(ServerAddr::Onion(onion_host), Service::Ssl(port)) => {
|
||||
format!("ssl://{}:{}", onion_host, port)
|
||||
}
|
||||
};
|
||||
|
||||
let mut config = electrum_client::ConfigBuilder::new();
|
||||
if let ServerAddr::Onion(_) = addr {
|
||||
let socks = electrum_client::Socks5Config::new(
|
||||
self.tor_proxy
|
||||
.chain_err(|| "no tor proxy configured, onion hosts are unsupported")?,
|
||||
);
|
||||
config = config.socks5(Some(socks))
|
||||
}
|
||||
|
||||
let client = Client::from_config(&server_url, config.build())?;
|
||||
|
||||
let features = client.server_features()?.try_into()?;
|
||||
self.verify_compatibility(&features)?;
|
||||
|
||||
if self.announce {
|
||||
// XXX should we require the other side to reciprocate?
|
||||
ensure!(
|
||||
client.server_add_peer(&self.our_features)?,
|
||||
"server does not reciprocate"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(features)
|
||||
}
|
||||
|
||||
fn verify_compatibility(&self, features: &ServerFeatures) -> Result<()> {
|
||||
ensure!(
|
||||
features.genesis_hash == self.our_features.genesis_hash,
|
||||
"incompatible networks"
|
||||
);
|
||||
|
||||
ensure!(
|
||||
features.protocol_min <= self.our_version && features.protocol_max >= self.our_version,
|
||||
"incompatible protocol versions"
|
||||
);
|
||||
|
||||
ensure!(
|
||||
features.hash_function == "sha256",
|
||||
"incompatible hash function"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn spawn_jobs_thread(manager: Arc<DiscoveryManager>) {
|
||||
spawn_thread("discovery-jobs", move || loop {
|
||||
if let Err(e) = manager.run_health_check() {
|
||||
debug!("health check failed: {:?}", e);
|
||||
}
|
||||
// XXX use a dynamic JOB_INTERVAL, adjusted according to the queue size and HEALTH_CHECK_FREQ?
|
||||
thread::sleep(JOB_INTERVAL);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl Server {
|
||||
fn new(hostname: Hostname, features: ServerFeatures) -> Self {
|
||||
Server {
|
||||
hostname,
|
||||
features,
|
||||
services: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get server features and services in the compact string array format used for `servers.peers.subscribe`
|
||||
fn feature_strs(&self) -> Vec<String> {
|
||||
let mut strs = Vec::with_capacity(self.services.len() + 1);
|
||||
strs.push(format!("v{}", self.features.protocol_max));
|
||||
if let Some(pruning) = self.features.pruning {
|
||||
strs.push(format!("p{}", pruning));
|
||||
}
|
||||
strs.extend(self.services.iter().map(|s| s.to_string()));
|
||||
strs
|
||||
}
|
||||
}
|
||||
|
||||
impl ServerAddr {
|
||||
fn resolve(host: &str) -> Result<Self> {
|
||||
Ok(if host.ends_with(".onion") {
|
||||
ServerAddr::Onion(host.into())
|
||||
} else if let Ok(ip) = IpAddr::from_str(host) {
|
||||
ServerAddr::Clearnet(ip)
|
||||
} else {
|
||||
let ip = format!("{}:1", host)
|
||||
.to_socket_addrs()
|
||||
.chain_err(|| "hostname resolution failed")?
|
||||
.next()
|
||||
.chain_err(|| "hostname resolution failed")?
|
||||
.ip();
|
||||
ServerAddr::Clearnet(ip)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ServerAddr {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
ServerAddr::Clearnet(ip) => write!(f, "{}", ip),
|
||||
ServerAddr::Onion(hostname) => write!(f, "{}", hostname),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl serde::Serialize for ServerAddr {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
serializer.serialize_str(&self.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthCheck {
|
||||
fn new(
|
||||
addr: ServerAddr,
|
||||
hostname: Hostname,
|
||||
service: Service,
|
||||
added_by: Option<IpAddr>,
|
||||
) -> Self {
|
||||
HealthCheck {
|
||||
addr,
|
||||
hostname,
|
||||
service,
|
||||
is_default: added_by.is_none(),
|
||||
added_by,
|
||||
last_check: None,
|
||||
last_healthy: None,
|
||||
consecutive_failures: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_healthy(&self) -> bool {
|
||||
match (self.last_check, self.last_healthy) {
|
||||
(Some(last_check), Some(last_healthy)) => last_check == last_healthy,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
// allow the server to fail up to MAX_CONSECTIVE_FAILURES time before giving up on it.
|
||||
// if its a non-default server and the very first attempt fails, give up immediatly.
|
||||
fn should_retry(&self) -> bool {
|
||||
(self.last_healthy.is_some() || self.is_default)
|
||||
&& self.consecutive_failures < MAX_CONSECUTIVE_FAILURES
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for HealthCheck {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.hostname == other.hostname && self.service == other.service
|
||||
}
|
||||
}
|
||||
|
||||
impl Ord for HealthCheck {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
self.last_check.cmp(&other.last_check).reverse()
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for HealthCheck {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Service {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Service::Tcp(port) => write!(f, "t{}", port),
|
||||
Service::Ssl(port) => write!(f, "s{}", port),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_remote_addr(addr: &ServerAddr) -> bool {
|
||||
match addr {
|
||||
ServerAddr::Onion(_) => true,
|
||||
ServerAddr::Clearnet(ip) => {
|
||||
!ip.is_loopback()
|
||||
&& !ip.is_unspecified()
|
||||
&& !ip.is_multicast()
|
||||
&& !match ip {
|
||||
IpAddr::V4(ipv4) => ipv4.is_private(),
|
||||
IpAddr::V6(_) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::chain::genesis_hash;
|
||||
use crate::chain::Network;
|
||||
use std::time;
|
||||
|
||||
use crate::config::VERSION_STRING;
|
||||
|
||||
const PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::new(1, 4);
|
||||
|
||||
#[test]
|
||||
#[ignore = "This test requires external connection to server that no longer exists"]
|
||||
fn test() -> Result<()> {
|
||||
stderrlog::new().verbosity(4).init().unwrap();
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
let testnet = Network::LiquidTestnet;
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
let testnet = Network::Testnet;
|
||||
|
||||
let features = ServerFeatures {
|
||||
hosts: serde_json::from_str("{\"test.foobar.example\":{\"tcp_port\":60002}}").unwrap(),
|
||||
server_version: VERSION_STRING.clone(),
|
||||
genesis_hash: genesis_hash(testnet),
|
||||
protocol_min: PROTOCOL_VERSION,
|
||||
protocol_max: PROTOCOL_VERSION,
|
||||
hash_function: "sha256".into(),
|
||||
pruning: None,
|
||||
};
|
||||
let discovery = Arc::new(DiscoveryManager::new(
|
||||
testnet,
|
||||
features,
|
||||
PROTOCOL_VERSION,
|
||||
false,
|
||||
None,
|
||||
));
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum.blockstream.info".into(),
|
||||
vec![Service::Tcp(60001)],
|
||||
)
|
||||
.unwrap();
|
||||
discovery
|
||||
.add_default_server("testnet.hsmiths.com".into(), vec![Service::Ssl(53012)])
|
||||
.unwrap();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"tn.not.fyi".into(),
|
||||
vec![Service::Tcp(55001), Service::Ssl(55002)],
|
||||
)
|
||||
.unwrap();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum.blockstream.info".into(),
|
||||
vec![Service::Tcp(60001), Service::Ssl(60002)],
|
||||
)
|
||||
.unwrap();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"explorerzydxu5ecjrkwceayqybizmpjjznk5izmitf2modhcusuqlid.onion".into(),
|
||||
vec![Service::Tcp(143)],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
debug!("{:#?}", discovery);
|
||||
|
||||
for _ in 0..12 {
|
||||
discovery
|
||||
.run_health_check()
|
||||
.map_err(|e| warn!("{:?}", e))
|
||||
.ok();
|
||||
thread::sleep(time::Duration::from_secs(1));
|
||||
}
|
||||
|
||||
debug!("{:#?}", discovery);
|
||||
|
||||
info!("{}", json!(discovery.get_servers()));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@ -1,449 +0,0 @@
|
||||
use crate::chain::Network;
|
||||
#[allow(unused_imports)]
|
||||
use crate::electrum::discovery::{DiscoveryManager, Service};
|
||||
|
||||
#[allow(unused_variables)]
|
||||
pub fn add_default_servers(discovery: &DiscoveryManager, network: Network) {
|
||||
match network {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Bitcoin => {
|
||||
discovery
|
||||
.add_default_server(
|
||||
"3smoooajg7qqac2y.onion".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"81-7-10-251.blue.kundencontroller.de".into(),
|
||||
vec![Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"E-X.not.fyi".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"VPS.hsmiths.com".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"b.ooze.cc".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"bauerjda5hnedjam.onion".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"bauerjhejlv6di7s.onion".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"bitcoin.corgi.party".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"bitcoin3nqy3db7c.onion".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"bitcoins.sk".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"btc.cihar.com".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"btc.xskyx.net".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"currentlane.lovebitco.in".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"daedalus.bauerj.eu".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum.jochen-hoenicke.de".into(),
|
||||
vec![Service::Tcp(50003), Service::Ssl(50005)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"dragon085.startdedicated.de".into(),
|
||||
vec![Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"e-1.claudioboxx.com".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"e.keff.org".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum-server.ninja".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum-unlimited.criptolayer.net".into(),
|
||||
vec![Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum.eff.ro".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum.festivaldelhumor.org".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum.hsmiths.com".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum.leblancnet.us".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("electrum.mindspot.org".into(), vec![Service::Ssl(50002)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum.qtornado.com".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("electrum.taborsky.cz".into(), vec![Service::Ssl(50002)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum.villocq.com".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum2.eff.ro".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum2.villocq.com".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrumx.bot.nu".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrumx.ddns.net".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("electrumx.ftp.sh".into(), vec![Service::Ssl(50002)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrumx.ml".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrumx.soon.it".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("electrumxhqdsmlu.onion".into(), vec![Service::Tcp(50001)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"elx01.knas.systems".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"enode.duckdns.org".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"fedaykin.goip.de".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"fn.48.org".into(),
|
||||
vec![Service::Tcp(50003), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"helicarrier.bauerj.eu".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"hsmiths4fyqlw5xw.onion".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"hsmiths5mjk6uijs.onion".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"icarus.tetradrachm.net".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrum.emzy.de".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"ndnd.selfhost.eu".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("ndndword5lpb7eex.onion".into(), vec![Service::Tcp(50001)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"orannis.com".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"ozahtqwp25chjdjd.onion".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"qtornadoklbgdyww.onion".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("rbx.curalle.ovh".into(), vec![Service::Ssl(50002)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("s7clinmo4cazmhul.onion".into(), vec![Service::Tcp(50001)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"tardis.bauerj.eu".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("technetium.network".into(), vec![Service::Ssl(50002)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"tomscryptos.com".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"ulrichard.ch".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"vmd27610.contaboserver.net".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"vmd30612.contaboserver.net".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"wsw6tua3xl24gsmi264zaep6seppjyrkyucpsmuxnjzyt3f3j6swshad.onion".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"xray587.startdedicated.de".into(),
|
||||
vec![Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"yuio.top".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"bitcoin.dragon.zone".into(),
|
||||
vec![Service::Tcp(50003), Service::Ssl(50004)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"ecdsa.net".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(110)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("btc.usebsv.com".into(), vec![Service::Ssl(50006)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"e2.keff.org".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("electrum.hodlister.co".into(), vec![Service::Ssl(50002)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("electrum3.hodlister.co".into(), vec![Service::Ssl(50002)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server("electrum5.hodlister.co".into(), vec![Service::Ssl(50002)])
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrumx.electricnewyear.net".into(),
|
||||
vec![Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"fortress.qtornado.com".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(443)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"green-gold.westeurope.cloudapp.azure.com".into(),
|
||||
vec![Service::Tcp(56001), Service::Ssl(56002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"electrumx.erbium.eu".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
}
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Network::Testnet => {
|
||||
discovery
|
||||
.add_default_server(
|
||||
"hsmithsxurybd7uh.onion".into(),
|
||||
vec![Service::Tcp(53011), Service::Ssl(53012)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"testnet.hsmiths.com".into(),
|
||||
vec![Service::Tcp(53011), Service::Ssl(53012)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"testnet.qtornado.com".into(),
|
||||
vec![Service::Tcp(51001), Service::Ssl(51002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"testnet1.bauerj.eu".into(),
|
||||
vec![Service::Tcp(50001), Service::Ssl(50002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"tn.not.fyi".into(),
|
||||
vec![Service::Tcp(55001), Service::Ssl(55002)],
|
||||
)
|
||||
.ok();
|
||||
discovery
|
||||
.add_default_server(
|
||||
"bitcoin.cluelessperson.com".into(),
|
||||
vec![Service::Tcp(51001), Service::Ssl(51002)],
|
||||
)
|
||||
.ok();
|
||||
}
|
||||
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
@ -1,119 +0,0 @@
|
||||
mod server;
|
||||
pub use server::RPC;
|
||||
|
||||
#[cfg(feature = "electrum-discovery")]
|
||||
mod client;
|
||||
#[cfg(feature = "electrum-discovery")]
|
||||
mod discovery;
|
||||
#[cfg(feature = "electrum-discovery")]
|
||||
pub use {client::Client, discovery::DiscoveryManager};
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::HashMap;
|
||||
use std::str::FromStr;
|
||||
|
||||
use serde::{de, Deserialize, Deserializer, Serialize};
|
||||
|
||||
use crate::chain::BlockHash;
|
||||
use crate::errors::ResultExt;
|
||||
use crate::util::BlockId;
|
||||
|
||||
pub fn get_electrum_height(blockid: Option<BlockId>, has_unconfirmed_parents: bool) -> isize {
|
||||
match (blockid, has_unconfirmed_parents) {
|
||||
(Some(blockid), _) => blockid.height as isize,
|
||||
(None, false) => 0,
|
||||
(None, true) => -1,
|
||||
}
|
||||
}
|
||||
|
||||
pub type Port = u16;
|
||||
pub type Hostname = String;
|
||||
|
||||
pub type ServerHosts = HashMap<Hostname, ServerPorts>;
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct ServerFeatures {
|
||||
pub hosts: ServerHosts,
|
||||
pub genesis_hash: BlockHash,
|
||||
pub server_version: String,
|
||||
pub protocol_min: ProtocolVersion,
|
||||
pub protocol_max: ProtocolVersion,
|
||||
pub pruning: Option<usize>,
|
||||
pub hash_function: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct ServerPorts {
|
||||
tcp_port: Option<Port>,
|
||||
ssl_port: Option<Port>,
|
||||
}
|
||||
|
||||
#[derive(Eq, PartialEq, Debug, Clone, Default)]
|
||||
pub struct ProtocolVersion {
|
||||
major: usize,
|
||||
minor: usize,
|
||||
}
|
||||
|
||||
impl ProtocolVersion {
|
||||
pub const fn new(major: usize, minor: usize) -> Self {
|
||||
Self { major, minor }
|
||||
}
|
||||
}
|
||||
|
||||
impl Ord for ProtocolVersion {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
self.major
|
||||
.cmp(&other.major)
|
||||
.then_with(|| self.minor.cmp(&other.minor))
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for ProtocolVersion {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for ProtocolVersion {
|
||||
type Err = crate::errors::Error;
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
let mut iter = s.split('.');
|
||||
Ok(Self {
|
||||
major: iter
|
||||
.next()
|
||||
.chain_err(|| "missing major")?
|
||||
.parse()
|
||||
.chain_err(|| "invalid major")?,
|
||||
minor: iter
|
||||
.next()
|
||||
.chain_err(|| "missing minor")?
|
||||
.parse()
|
||||
.chain_err(|| "invalid minor")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ProtocolVersion {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}.{}", self.major, self.minor)
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for ProtocolVersion {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
serializer.collect_str(&self)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for ProtocolVersion {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let s = String::deserialize(deserializer)?;
|
||||
FromStr::from_str(&s).map_err(de::Error::custom)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,662 +0,0 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{Arc, RwLock, RwLockReadGuard};
|
||||
|
||||
use bitcoin::hashes::{sha256, Hash};
|
||||
use elements::confidential::{Asset, Value};
|
||||
use elements::encode::{deserialize, serialize};
|
||||
use elements::secp256k1_zkp::ZERO_TWEAK;
|
||||
use elements::{issuance::ContractHash, AssetId, AssetIssuance, OutPoint, Transaction, TxIn};
|
||||
|
||||
use crate::chain::{BNetwork, BlockHash, Network, Txid};
|
||||
use crate::elements::peg::{get_pegin_data, get_pegout_data, PeginInfo, PegoutInfo};
|
||||
use crate::elements::registry::{AssetMeta, AssetRegistry};
|
||||
use crate::errors::*;
|
||||
use crate::new_index::schema::{Operation, TxHistoryInfo, TxHistoryKey, TxHistoryRow};
|
||||
use crate::new_index::{db::DBFlush, ChainQuery, DBRow, Mempool, Query};
|
||||
use crate::util::{
|
||||
bincode_util, full_hash, Bytes, FullHash, IsProvablyUnspendable, TransactionStatus, TxInput,
|
||||
};
|
||||
|
||||
lazy_static! {
|
||||
pub static ref NATIVE_ASSET_ID: AssetId =
|
||||
"6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d"
|
||||
.parse()
|
||||
.unwrap();
|
||||
pub static ref NATIVE_ASSET_ID_TESTNET: AssetId =
|
||||
"144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49"
|
||||
.parse()
|
||||
.unwrap();
|
||||
pub static ref NATIVE_ASSET_ID_REGTEST: AssetId =
|
||||
"5ac9f65c0efcc4775e0baec4ec03abdde22473cd3cf33c0419ca290e0751b225"
|
||||
.parse()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn parse_asset_id(sl: &[u8]) -> AssetId {
|
||||
AssetId::from_slice(sl).expect("failed to parse AssetId")
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(untagged)]
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
pub enum LiquidAsset {
|
||||
Issued(IssuedAsset),
|
||||
Native(PeggedAsset),
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct PeggedAsset {
|
||||
pub asset_id: AssetId,
|
||||
pub chain_stats: PeggedAssetStats,
|
||||
pub mempool_stats: PeggedAssetStats,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct IssuedAsset {
|
||||
pub asset_id: AssetId,
|
||||
pub issuance_txin: TxInput,
|
||||
#[serde(serialize_with = "crate::util::serialize_outpoint")]
|
||||
pub issuance_prevout: OutPoint,
|
||||
pub reissuance_token: AssetId,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub contract_hash: Option<ContractHash>,
|
||||
|
||||
// the confirmation status of the initial issuance transaction
|
||||
pub status: TransactionStatus,
|
||||
|
||||
pub chain_stats: IssuedAssetStats,
|
||||
pub mempool_stats: IssuedAssetStats,
|
||||
|
||||
// optional metadata from registry
|
||||
#[serde(flatten)]
|
||||
pub meta: Option<AssetMeta>,
|
||||
}
|
||||
|
||||
// DB representation (issued assets only)
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct AssetRow {
|
||||
pub issuance_txid: FullHash,
|
||||
pub issuance_vin: u32,
|
||||
pub prev_txid: FullHash,
|
||||
pub prev_vout: u32,
|
||||
pub issuance: Bytes, // bincode does not like dealing with AssetIssuance, deserialization fails with "invalid type: sequence, expected a struct"
|
||||
pub reissuance_token: FullHash,
|
||||
}
|
||||
|
||||
impl IssuedAsset {
|
||||
pub fn new(
|
||||
asset_id: &AssetId,
|
||||
asset: &AssetRow,
|
||||
(chain_stats, mempool_stats): (IssuedAssetStats, IssuedAssetStats),
|
||||
meta: Option<AssetMeta>,
|
||||
status: TransactionStatus,
|
||||
) -> Self {
|
||||
let issuance: AssetIssuance =
|
||||
deserialize(&asset.issuance).expect("failed parsing AssetIssuance");
|
||||
|
||||
let reissuance_token = parse_asset_id(&asset.reissuance_token);
|
||||
|
||||
let contract_hash = if issuance.asset_entropy != [0u8; 32] {
|
||||
Some(ContractHash::from_byte_array(issuance.asset_entropy))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Self {
|
||||
asset_id: *asset_id,
|
||||
issuance_txin: TxInput {
|
||||
txid: deserialize(&asset.issuance_txid).unwrap(),
|
||||
vin: asset.issuance_vin,
|
||||
},
|
||||
issuance_prevout: OutPoint {
|
||||
txid: deserialize(&asset.prev_txid).unwrap(),
|
||||
vout: asset.prev_vout,
|
||||
},
|
||||
contract_hash,
|
||||
reissuance_token,
|
||||
status,
|
||||
chain_stats,
|
||||
mempool_stats,
|
||||
meta,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LiquidAsset {
|
||||
pub fn supply(&self) -> Option<u64> {
|
||||
match self {
|
||||
LiquidAsset::Native(asset) => Some(
|
||||
asset.chain_stats.peg_in_amount
|
||||
- asset.chain_stats.peg_out_amount
|
||||
- asset.chain_stats.burned_amount
|
||||
+ asset.mempool_stats.peg_in_amount
|
||||
- asset.mempool_stats.peg_out_amount
|
||||
- asset.mempool_stats.burned_amount,
|
||||
),
|
||||
LiquidAsset::Issued(asset) => {
|
||||
if asset.chain_stats.has_blinded_issuances
|
||||
|| asset.mempool_stats.has_blinded_issuances
|
||||
{
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
asset.chain_stats.issued_amount - asset.chain_stats.burned_amount
|
||||
+ asset.mempool_stats.issued_amount
|
||||
- asset.mempool_stats.burned_amount,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
pub fn precision(&self) -> u8 {
|
||||
match self {
|
||||
LiquidAsset::Native(_) => 8,
|
||||
LiquidAsset::Issued(asset) => asset.meta.as_ref().map_or(0, |m| m.precision),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[cfg_attr(test, derive(PartialEq, Eq))]
|
||||
pub struct IssuingInfo {
|
||||
pub txid: FullHash,
|
||||
pub vin: u32,
|
||||
pub is_reissuance: bool,
|
||||
// None for blinded issuances
|
||||
pub issued_amount: Option<u64>,
|
||||
pub token_amount: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[cfg_attr(test, derive(PartialEq, Eq))]
|
||||
pub struct BurningInfo {
|
||||
pub txid: FullHash,
|
||||
pub vout: u32,
|
||||
pub value: u64,
|
||||
}
|
||||
|
||||
// Index confirmed transaction issuances and save as db rows
|
||||
pub fn index_confirmed_tx_assets(
|
||||
tx: &Transaction,
|
||||
confirmed_height: u32,
|
||||
tx_position: u16,
|
||||
network: Network,
|
||||
parent_network: BNetwork,
|
||||
rows: &mut Vec<DBRow>,
|
||||
op: &Operation,
|
||||
) {
|
||||
let (history, issuances) = index_tx_assets(tx, network, parent_network);
|
||||
|
||||
rows.extend(history.into_iter().map(|(asset_id, info)| {
|
||||
let history_row = asset_history_row(&asset_id, confirmed_height, tx_position, info);
|
||||
if let Operation::DeleteBlocksWithHistory(tx) = op {
|
||||
tx.send(history_row.key.hash)
|
||||
.expect("unbounded channel won't fail");
|
||||
}
|
||||
history_row.into_row()
|
||||
}));
|
||||
|
||||
// the initial issuance is kept twice: once in the history index under I<asset><height><txid:vin>,
|
||||
// and once separately under i<asset> for asset lookup with some more associated metadata.
|
||||
// reissuances are only kept under the history index.
|
||||
rows.extend(issuances.into_iter().map(|(asset_id, asset_row)| DBRow {
|
||||
key: [b"i", &asset_id.into_inner()[..]].concat(),
|
||||
value: bincode_util::serialize_little(&asset_row).unwrap(),
|
||||
}));
|
||||
}
|
||||
|
||||
// Index mempool transaction issuances and save to in-memory store
|
||||
pub fn index_mempool_tx_assets(
|
||||
tx: &Transaction,
|
||||
network: Network,
|
||||
parent_network: BNetwork,
|
||||
asset_history: &mut HashMap<AssetId, Vec<TxHistoryInfo>>,
|
||||
asset_issuance: &mut HashMap<AssetId, AssetRow>,
|
||||
) {
|
||||
let (history, issuances) = index_tx_assets(tx, network, parent_network);
|
||||
for (asset_id, info) in history {
|
||||
asset_history.entry(asset_id).or_default().push(info);
|
||||
}
|
||||
for (asset_id, issuance) in issuances {
|
||||
asset_issuance.insert(asset_id, issuance);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove mempool transaction issuances from in-memory store
|
||||
pub fn remove_mempool_tx_assets(
|
||||
to_remove: &HashSet<&Txid>,
|
||||
asset_history: &mut HashMap<AssetId, Vec<TxHistoryInfo>>,
|
||||
asset_issuance: &mut HashMap<AssetId, AssetRow>,
|
||||
) {
|
||||
// TODO optimize
|
||||
asset_history.retain(|_assethash, entries| {
|
||||
entries.retain(|entry| !to_remove.contains(&entry.get_txid()));
|
||||
!entries.is_empty()
|
||||
});
|
||||
|
||||
asset_issuance.retain(|_assethash, issuance| {
|
||||
let txid: Txid = deserialize(&issuance.issuance_txid).unwrap();
|
||||
!to_remove.contains(&txid)
|
||||
});
|
||||
}
|
||||
|
||||
type HistoryAndIssuances = (Vec<(AssetId, TxHistoryInfo)>, Vec<(AssetId, AssetRow)>);
|
||||
// Internal utility function, index a transaction and return its history entries and issuances
|
||||
fn index_tx_assets(
|
||||
tx: &Transaction,
|
||||
network: Network,
|
||||
parent_network: BNetwork,
|
||||
) -> HistoryAndIssuances {
|
||||
let mut history = vec![];
|
||||
let mut issuances = vec![];
|
||||
|
||||
let txid = full_hash(&tx.txid()[..]);
|
||||
|
||||
for (txo_index, txo) in tx.output.iter().enumerate() {
|
||||
if let Some(pegout) = get_pegout_data(txo, network, parent_network) {
|
||||
history.push((
|
||||
pegout.asset.explicit().unwrap(),
|
||||
TxHistoryInfo::Pegout(PegoutInfo {
|
||||
txid,
|
||||
vout: txo_index as u32,
|
||||
value: pegout.value,
|
||||
}),
|
||||
));
|
||||
} else if txo.script_pubkey.is_provably_unspendable_() && !txo.is_fee() {
|
||||
if let (Asset::Explicit(asset_id), Value::Explicit(value)) = (txo.asset, txo.value) {
|
||||
if value > 0 {
|
||||
history.push((
|
||||
asset_id,
|
||||
TxHistoryInfo::Burning(BurningInfo {
|
||||
txid,
|
||||
vout: txo_index as u32,
|
||||
value,
|
||||
}),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (txi_index, txi) in tx.input.iter().enumerate() {
|
||||
if let Some(pegin) = get_pegin_data(txi, network) {
|
||||
history.push((
|
||||
pegin.asset,
|
||||
TxHistoryInfo::Pegin(PeginInfo {
|
||||
txid,
|
||||
vin: txi_index as u32,
|
||||
value: pegin.value,
|
||||
}),
|
||||
));
|
||||
} else if txi.has_issuance() {
|
||||
let is_reissuance = txi.asset_issuance.asset_blinding_nonce != ZERO_TWEAK;
|
||||
|
||||
let asset_entropy = get_issuance_entropy(txi).expect("invalid issuance");
|
||||
let asset_id = AssetId::from_entropy(asset_entropy);
|
||||
|
||||
let issued_amount = match txi.asset_issuance.amount {
|
||||
Value::Explicit(amount) => Some(amount),
|
||||
Value::Null => Some(0),
|
||||
_ => None,
|
||||
};
|
||||
let token_amount = match txi.asset_issuance.inflation_keys {
|
||||
Value::Explicit(amount) => Some(amount),
|
||||
Value::Null => Some(0),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
history.push((
|
||||
asset_id,
|
||||
TxHistoryInfo::Issuing(IssuingInfo {
|
||||
txid,
|
||||
vin: txi_index as u32,
|
||||
is_reissuance,
|
||||
issued_amount,
|
||||
token_amount,
|
||||
}),
|
||||
));
|
||||
|
||||
if !is_reissuance {
|
||||
let is_confidential =
|
||||
matches!(txi.asset_issuance.inflation_keys, Value::Confidential(..));
|
||||
let reissuance_token =
|
||||
AssetId::reissuance_token_from_entropy(asset_entropy, is_confidential);
|
||||
|
||||
issuances.push((
|
||||
asset_id,
|
||||
AssetRow {
|
||||
issuance_txid: txid,
|
||||
issuance_vin: txi_index as u32,
|
||||
prev_txid: full_hash(&txi.previous_output.txid[..]),
|
||||
prev_vout: txi.previous_output.vout,
|
||||
issuance: serialize(&txi.asset_issuance),
|
||||
reissuance_token: full_hash(&reissuance_token.into_inner()[..]),
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(history, issuances)
|
||||
}
|
||||
|
||||
fn asset_history_row(
|
||||
asset_id: &AssetId,
|
||||
confirmed_height: u32,
|
||||
tx_position: u16,
|
||||
txinfo: TxHistoryInfo,
|
||||
) -> TxHistoryRow {
|
||||
let key = TxHistoryKey {
|
||||
code: b'I',
|
||||
hash: full_hash(&asset_id.into_inner()[..]),
|
||||
confirmed_height,
|
||||
tx_position,
|
||||
txinfo,
|
||||
};
|
||||
TxHistoryRow { key }
|
||||
}
|
||||
|
||||
pub enum AssetRegistryLock<'a> {
|
||||
RwLock(&'a Arc<RwLock<AssetRegistry>>),
|
||||
RwLockReadGuard(&'a RwLockReadGuard<'a, AssetRegistry>),
|
||||
}
|
||||
|
||||
pub fn lookup_asset(
|
||||
query: &Query,
|
||||
registry: Option<AssetRegistryLock>,
|
||||
asset_id: &AssetId,
|
||||
meta: Option<&AssetMeta>, // may optionally be provided if already known
|
||||
) -> Result<Option<LiquidAsset>> {
|
||||
if query.network().pegged_asset() == Some(asset_id) {
|
||||
let (chain_stats, mempool_stats) = pegged_asset_stats(query, asset_id);
|
||||
|
||||
return Ok(Some(LiquidAsset::Native(PeggedAsset {
|
||||
asset_id: *asset_id,
|
||||
chain_stats,
|
||||
mempool_stats,
|
||||
})));
|
||||
}
|
||||
|
||||
let history_db = query.chain().store().history_db();
|
||||
let mempool = query.mempool();
|
||||
let mempool_issuances = &mempool.asset_issuance;
|
||||
|
||||
let chain_row = history_db
|
||||
.get(&[b"i", &asset_id.into_inner()[..]].concat())
|
||||
.map(|row| {
|
||||
bincode_util::deserialize_little::<AssetRow>(&row).expect("failed parsing AssetRow")
|
||||
});
|
||||
|
||||
let row = chain_row
|
||||
.as_ref()
|
||||
.or_else(|| mempool_issuances.get(asset_id));
|
||||
|
||||
Ok(if let Some(row) = row {
|
||||
let reissuance_token = parse_asset_id(&row.reissuance_token);
|
||||
|
||||
let meta = meta.cloned().or_else(|| match registry {
|
||||
Some(AssetRegistryLock::RwLock(rwlock)) => {
|
||||
rwlock.read().unwrap().get(asset_id).cloned()
|
||||
}
|
||||
Some(AssetRegistryLock::RwLockReadGuard(guard)) => guard.get(asset_id).cloned(),
|
||||
None => None,
|
||||
});
|
||||
let stats = issued_asset_stats(query.chain(), &mempool, asset_id, &reissuance_token);
|
||||
let status = query.get_tx_status(&deserialize(&row.issuance_txid).unwrap());
|
||||
|
||||
let asset = IssuedAsset::new(asset_id, row, stats, meta, status);
|
||||
|
||||
Some(LiquidAsset::Issued(asset))
|
||||
} else {
|
||||
None
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_issuance_entropy(txin: &TxIn) -> Result<sha256::Midstate> {
|
||||
if !txin.has_issuance() {
|
||||
bail!("input has no issuance");
|
||||
}
|
||||
|
||||
let is_reissuance = txin.asset_issuance.asset_blinding_nonce != ZERO_TWEAK;
|
||||
|
||||
Ok(if !is_reissuance {
|
||||
let contract_hash = ContractHash::from_slice(&txin.asset_issuance.asset_entropy)
|
||||
.chain_err(|| "invalid entropy (contract hash)")?;
|
||||
AssetId::generate_asset_entropy(txin.previous_output, contract_hash)
|
||||
} else {
|
||||
sha256::Midstate::from_slice(&txin.asset_issuance.asset_entropy)
|
||||
.chain_err(|| "invalid entropy (reissuance)")?
|
||||
})
|
||||
}
|
||||
|
||||
//
|
||||
// Asset stats
|
||||
//
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Default)]
|
||||
pub struct IssuedAssetStats {
|
||||
pub tx_count: usize,
|
||||
pub issuance_count: usize,
|
||||
pub issued_amount: u64,
|
||||
pub burned_amount: u64,
|
||||
pub has_blinded_issuances: bool,
|
||||
pub reissuance_tokens: Option<u64>, // none if confidential
|
||||
pub burned_reissuance_tokens: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Default)]
|
||||
pub struct PeggedAssetStats {
|
||||
pub tx_count: usize,
|
||||
pub peg_in_count: usize,
|
||||
pub peg_in_amount: u64,
|
||||
pub peg_out_count: usize,
|
||||
pub peg_out_amount: u64,
|
||||
pub burn_count: usize,
|
||||
pub burned_amount: u64,
|
||||
}
|
||||
|
||||
type AssetStatApplyFn<T> = fn(&TxHistoryInfo, &mut T, &mut HashSet<Txid>);
|
||||
|
||||
fn asset_cache_key(asset_id: &AssetId) -> Bytes {
|
||||
[b"z", &asset_id.into_inner()[..]].concat()
|
||||
}
|
||||
|
||||
fn asset_cache_row<T>(asset_id: &AssetId, stats: &T, blockhash: &BlockHash) -> DBRow
|
||||
where
|
||||
T: serde::Serialize,
|
||||
{
|
||||
DBRow {
|
||||
key: asset_cache_key(asset_id),
|
||||
value: bincode_util::serialize_little(&(stats, blockhash)).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
// Get stats for the network's pegged asset
|
||||
fn pegged_asset_stats(query: &Query, asset_id: &AssetId) -> (PeggedAssetStats, PeggedAssetStats) {
|
||||
(
|
||||
chain_asset_stats(query.chain(), asset_id, apply_pegged_asset_stats),
|
||||
mempool_asset_stats(&query.mempool(), asset_id, apply_pegged_asset_stats),
|
||||
)
|
||||
}
|
||||
|
||||
// Get stats for issued assets
|
||||
fn issued_asset_stats(
|
||||
chain: &ChainQuery,
|
||||
mempool: &Mempool,
|
||||
asset_id: &AssetId,
|
||||
reissuance_token: &AssetId,
|
||||
) -> (IssuedAssetStats, IssuedAssetStats) {
|
||||
let afn = apply_issued_asset_stats;
|
||||
|
||||
let mut chain_stats = chain_asset_stats(chain, asset_id, afn);
|
||||
chain_stats.burned_reissuance_tokens =
|
||||
chain_asset_stats(chain, reissuance_token, afn).burned_amount;
|
||||
|
||||
let mut mempool_stats = mempool_asset_stats(mempool, asset_id, afn);
|
||||
mempool_stats.burned_reissuance_tokens =
|
||||
mempool_asset_stats(mempool, reissuance_token, afn).burned_amount;
|
||||
|
||||
(chain_stats, mempool_stats)
|
||||
}
|
||||
|
||||
// Get on-chain confirmed asset stats (issued or the pegged asset)
|
||||
fn chain_asset_stats<T>(chain: &ChainQuery, asset_id: &AssetId, apply_fn: AssetStatApplyFn<T>) -> T
|
||||
where
|
||||
T: Default + serde::Serialize + serde::de::DeserializeOwned,
|
||||
{
|
||||
// get the last known stats and the blockhash they are updated for.
|
||||
// invalidates the cache if the block was orphaned.
|
||||
let cache: Option<(T, usize)> = chain
|
||||
.store()
|
||||
.cache_db()
|
||||
.get(&asset_cache_key(asset_id))
|
||||
.map(|c| bincode_util::deserialize_little(&c).unwrap())
|
||||
.and_then(|(stats, blockhash)| {
|
||||
chain
|
||||
.height_by_hash(&blockhash)
|
||||
.map(|height| (stats, height))
|
||||
});
|
||||
|
||||
// update stats with new transactions since
|
||||
let (newstats, lastblock) = cache.map_or_else(
|
||||
|| chain_asset_stats_delta(chain, asset_id, T::default(), 0, apply_fn),
|
||||
|(oldstats, blockheight)| {
|
||||
chain_asset_stats_delta(chain, asset_id, oldstats, blockheight + 1, apply_fn)
|
||||
},
|
||||
);
|
||||
|
||||
// save updated stats to cache
|
||||
if let Some(lastblock) = lastblock {
|
||||
chain.store().cache_db().write(
|
||||
vec![asset_cache_row(asset_id, &newstats, &lastblock)],
|
||||
DBFlush::Enable,
|
||||
);
|
||||
}
|
||||
|
||||
newstats
|
||||
}
|
||||
|
||||
// Update the asset stats with the delta of confirmed txs since start_height
|
||||
fn chain_asset_stats_delta<T>(
|
||||
chain: &ChainQuery,
|
||||
asset_id: &AssetId,
|
||||
init_stats: T,
|
||||
start_height: usize,
|
||||
apply_fn: AssetStatApplyFn<T>,
|
||||
) -> (T, Option<BlockHash>) {
|
||||
let history_iter = chain
|
||||
.history_iter_scan(b'I', &asset_id.into_inner()[..], start_height)
|
||||
.map(TxHistoryRow::from_row)
|
||||
.filter_map(|history| {
|
||||
chain
|
||||
.tx_confirming_block(&history.get_txid())
|
||||
.map(|blockid| (history, blockid))
|
||||
});
|
||||
|
||||
let mut stats = init_stats;
|
||||
let mut seen_txids = HashSet::new();
|
||||
let mut lastblock = None;
|
||||
|
||||
for (row, blockid) in history_iter {
|
||||
if lastblock != Some(blockid.hash) {
|
||||
seen_txids.clear();
|
||||
}
|
||||
apply_fn(&row.key.txinfo, &mut stats, &mut seen_txids);
|
||||
lastblock = Some(blockid.hash);
|
||||
}
|
||||
|
||||
(stats, lastblock)
|
||||
}
|
||||
|
||||
// Get mempool asset stats (issued or the pegged asset)
|
||||
pub fn mempool_asset_stats<T>(
|
||||
mempool: &Mempool,
|
||||
asset_id: &AssetId,
|
||||
apply_fn: AssetStatApplyFn<T>,
|
||||
) -> T
|
||||
where
|
||||
T: Default,
|
||||
{
|
||||
let mut stats = T::default();
|
||||
|
||||
if let Some(history) = mempool.asset_history.get(asset_id) {
|
||||
let mut seen_txids = HashSet::new();
|
||||
for info in history {
|
||||
apply_fn(info, &mut stats, &mut seen_txids)
|
||||
}
|
||||
}
|
||||
|
||||
stats
|
||||
}
|
||||
|
||||
fn apply_issued_asset_stats(
|
||||
info: &TxHistoryInfo,
|
||||
stats: &mut IssuedAssetStats,
|
||||
seen_txids: &mut HashSet<Txid>,
|
||||
) {
|
||||
if seen_txids.insert(info.get_txid()) {
|
||||
stats.tx_count += 1;
|
||||
}
|
||||
|
||||
match info {
|
||||
TxHistoryInfo::Issuing(issuance) => {
|
||||
stats.issuance_count += 1;
|
||||
|
||||
match issuance.issued_amount {
|
||||
Some(amount) => stats.issued_amount += amount,
|
||||
None => stats.has_blinded_issuances = true,
|
||||
}
|
||||
|
||||
if !issuance.is_reissuance {
|
||||
stats.reissuance_tokens = issuance.token_amount;
|
||||
}
|
||||
}
|
||||
|
||||
TxHistoryInfo::Burning(info) => {
|
||||
stats.burned_amount += info.value;
|
||||
}
|
||||
|
||||
TxHistoryInfo::Funding(_) | TxHistoryInfo::Spending(_) => {
|
||||
// we don't keep funding/spending entries for assets
|
||||
unreachable!();
|
||||
}
|
||||
|
||||
TxHistoryInfo::Pegin(_) | TxHistoryInfo::Pegout(_) => {
|
||||
// issued assets cannot have pegins/pegouts
|
||||
unreachable!();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_pegged_asset_stats(
|
||||
info: &TxHistoryInfo,
|
||||
stats: &mut PeggedAssetStats,
|
||||
seen_txids: &mut HashSet<Txid>,
|
||||
) {
|
||||
if seen_txids.insert(info.get_txid()) {
|
||||
stats.tx_count += 1;
|
||||
}
|
||||
|
||||
match info {
|
||||
TxHistoryInfo::Pegin(info) => {
|
||||
stats.peg_in_count += 1;
|
||||
stats.peg_in_amount += info.value;
|
||||
}
|
||||
TxHistoryInfo::Pegout(info) => {
|
||||
stats.peg_out_count += 1;
|
||||
stats.peg_out_amount += info.value;
|
||||
}
|
||||
TxHistoryInfo::Burning(info) => {
|
||||
stats.burn_count += 1;
|
||||
stats.burned_amount += info.value;
|
||||
}
|
||||
TxHistoryInfo::Issuing(_) => {
|
||||
warn!("encountered issuance of native asset, ignoring (possibly freeinitialcoins?)");
|
||||
}
|
||||
TxHistoryInfo::Funding(_) | TxHistoryInfo::Spending(_) => {
|
||||
// these history entries variants are never kept for native assets
|
||||
unreachable!();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,77 +0,0 @@
|
||||
use bitcoin::hashes::Hash;
|
||||
use elements::hex::ToHex;
|
||||
use elements::secp256k1_zkp::ZERO_TWEAK;
|
||||
use elements::{confidential::Value, encode::serialize, issuance::ContractHash, AssetId, TxIn};
|
||||
|
||||
pub mod asset;
|
||||
pub mod peg;
|
||||
mod registry;
|
||||
|
||||
use asset::get_issuance_entropy;
|
||||
pub use asset::{lookup_asset, LiquidAsset};
|
||||
pub use registry::{AssetMeta, AssetRegistry, AssetSorting};
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
pub struct IssuanceValue {
|
||||
pub asset_id: String,
|
||||
pub is_reissuance: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub asset_blinding_nonce: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub contract_hash: Option<String>,
|
||||
pub asset_entropy: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub assetamount: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub assetamountcommitment: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tokenamount: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tokenamountcommitment: Option<String>,
|
||||
}
|
||||
|
||||
impl From<&TxIn> for IssuanceValue {
|
||||
fn from(txin: &TxIn) -> Self {
|
||||
let issuance = &txin.asset_issuance;
|
||||
let is_reissuance = issuance.asset_blinding_nonce != ZERO_TWEAK;
|
||||
|
||||
let asset_entropy = get_issuance_entropy(txin).expect("invalid issuance");
|
||||
let asset_id = AssetId::from_entropy(asset_entropy);
|
||||
|
||||
let contract_hash = if !is_reissuance {
|
||||
Some(ContractHash::from_slice(&issuance.asset_entropy).expect("invalid asset entropy"))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
IssuanceValue {
|
||||
asset_id: asset_id.to_hex(),
|
||||
asset_entropy: asset_entropy.to_hex(),
|
||||
contract_hash: contract_hash.map(|h| h.to_hex()),
|
||||
is_reissuance,
|
||||
asset_blinding_nonce: if is_reissuance {
|
||||
Some(hex::encode(issuance.asset_blinding_nonce.as_ref()))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
assetamount: match issuance.amount {
|
||||
Value::Explicit(value) => Some(value),
|
||||
Value::Null => Some(0),
|
||||
Value::Confidential(..) => None,
|
||||
},
|
||||
assetamountcommitment: match issuance.amount {
|
||||
Value::Confidential(..) => Some(hex::encode(serialize(&issuance.amount))),
|
||||
_ => None,
|
||||
},
|
||||
tokenamount: match issuance.inflation_keys {
|
||||
Value::Explicit(value) => Some(value),
|
||||
Value::Null => Some(0),
|
||||
Value::Confidential(..) => None,
|
||||
},
|
||||
tokenamountcommitment: match issuance.inflation_keys {
|
||||
Value::Confidential(..) => Some(hex::encode(serialize(&issuance.inflation_keys))),
|
||||
_ => None,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,84 +0,0 @@
|
||||
use elements::hex::ToHex;
|
||||
use elements::{confidential::Asset, PeginData, PegoutData, TxIn, TxOut};
|
||||
|
||||
use crate::chain::{bitcoin_genesis_hash, BNetwork, Network};
|
||||
use crate::util::FullHash;
|
||||
|
||||
pub fn get_pegin_data(txout: &TxIn, network: Network) -> Option<PeginData<'_>> {
|
||||
let pegged_asset_id = network.pegged_asset()?;
|
||||
txout.pegin_data().and_then(|pegin| {
|
||||
if pegin.asset == *pegged_asset_id {
|
||||
Some(pegin)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_pegout_data(
|
||||
txout: &TxOut,
|
||||
network: Network,
|
||||
parent_network: BNetwork,
|
||||
) -> Option<PegoutData<'_>> {
|
||||
let pegged_asset_id = network.pegged_asset()?;
|
||||
txout.pegout_data().and_then(|pegout| {
|
||||
if pegout.asset == Asset::Explicit(*pegged_asset_id)
|
||||
&& pegout.genesis_hash
|
||||
== bitcoin_genesis_hash(match parent_network {
|
||||
BNetwork::Bitcoin => Network::Liquid,
|
||||
BNetwork::Testnet | BNetwork::Testnet4 => Network::LiquidTestnet,
|
||||
BNetwork::Signet => return None,
|
||||
BNetwork::Regtest => Network::LiquidRegtest,
|
||||
})
|
||||
{
|
||||
Some(pegout)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// API representation of pegout data assocaited with an output
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
pub struct PegoutValue {
|
||||
pub genesis_hash: String,
|
||||
pub scriptpubkey: bitcoin::ScriptBuf,
|
||||
pub scriptpubkey_asm: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub scriptpubkey_address: Option<String>,
|
||||
}
|
||||
|
||||
impl PegoutValue {
|
||||
pub fn from_txout(txout: &TxOut, network: Network, parent_network: BNetwork) -> Option<Self> {
|
||||
let pegoutdata = get_pegout_data(txout, network, parent_network)?;
|
||||
|
||||
// pending https://github.com/ElementsProject/rust-elements/pull/69 is merged
|
||||
let scriptpubkey = bitcoin::ScriptBuf::from(pegoutdata.script_pubkey.into_bytes());
|
||||
let address = bitcoin::Address::from_script(&scriptpubkey, parent_network).ok();
|
||||
|
||||
Some(PegoutValue {
|
||||
genesis_hash: pegoutdata.genesis_hash.to_hex(),
|
||||
scriptpubkey_asm: scriptpubkey.to_asm_string(),
|
||||
scriptpubkey_address: address.map(|s| s.to_string()),
|
||||
scriptpubkey,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Inner type for the indexer TxHistoryInfo::Pegin variant
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[cfg_attr(test, derive(PartialEq, Eq))]
|
||||
pub struct PeginInfo {
|
||||
pub txid: FullHash,
|
||||
pub vin: u32,
|
||||
pub value: u64,
|
||||
}
|
||||
|
||||
// Inner type for the indexer TxHistoryInfo::Pegout variant
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[cfg_attr(test, derive(PartialEq, Eq))]
|
||||
pub struct PegoutInfo {
|
||||
pub txid: FullHash,
|
||||
pub vout: u32,
|
||||
pub value: u64,
|
||||
}
|
||||
@ -1,298 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
use std::str::FromStr;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::{cmp, fs, path, thread};
|
||||
|
||||
use serde_json::Value as JsonValue;
|
||||
|
||||
use elements::AssetId;
|
||||
|
||||
use crate::errors::*;
|
||||
|
||||
// length of asset id prefix to use for sub-directory partitioning
|
||||
// (in number of hex characters, not bytes)
|
||||
|
||||
const DIR_PARTITION_LEN: usize = 2;
|
||||
const SEARCH_SORT_CANDIDATE_LIMIT: usize = 2000;
|
||||
|
||||
pub struct AssetRegistry {
|
||||
directory: path::PathBuf,
|
||||
assets_cache: HashMap<AssetId, (SystemTime, AssetMeta)>,
|
||||
}
|
||||
|
||||
pub type AssetEntry<'a> = (&'a AssetId, &'a AssetMeta);
|
||||
|
||||
impl AssetRegistry {
|
||||
pub fn new(directory: path::PathBuf) -> Self {
|
||||
Self {
|
||||
directory,
|
||||
assets_cache: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get(&self, asset_id: &AssetId) -> Option<&AssetMeta> {
|
||||
self.assets_cache
|
||||
.get(asset_id)
|
||||
.map(|(_, metadata)| metadata)
|
||||
}
|
||||
|
||||
pub fn list(
|
||||
&self,
|
||||
start_index: usize,
|
||||
limit: usize,
|
||||
sorting: AssetSorting,
|
||||
) -> (usize, Vec<AssetEntry<'_>>) {
|
||||
let mut assets: Vec<AssetEntry> = self
|
||||
.assets_cache
|
||||
.iter()
|
||||
.map(|(asset_id, (_, metadata))| (asset_id, metadata))
|
||||
.collect();
|
||||
assets.sort_by(sorting.as_comparator());
|
||||
(
|
||||
assets.len(),
|
||||
assets.into_iter().skip(start_index).take(limit).collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn search(&self, query: &str, limit: usize) -> Vec<AssetEntry<'_>> {
|
||||
let query = query.trim();
|
||||
if query.is_empty() || limit == 0 {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
let (mut results, candidates) = search_by(
|
||||
self.assets_cache
|
||||
.iter()
|
||||
.map(|(asset_id, (_, metadata))| (asset_id, metadata)),
|
||||
query,
|
||||
limit,
|
||||
|metadata| metadata.ticker.as_deref(),
|
||||
);
|
||||
|
||||
if results.len() < limit {
|
||||
let (name_matches, candidates) =
|
||||
search_by(candidates, query, limit - results.len(), |metadata| {
|
||||
Some(&metadata.name)
|
||||
});
|
||||
results.extend(name_matches);
|
||||
|
||||
if results.len() < limit {
|
||||
let (domain_matches, _) =
|
||||
search_by(candidates, query, limit - results.len(), AssetMeta::domain);
|
||||
results.extend(domain_matches);
|
||||
}
|
||||
}
|
||||
|
||||
results.truncate(limit);
|
||||
results
|
||||
}
|
||||
|
||||
pub fn fs_sync(&mut self) -> Result<()> {
|
||||
for entry in fs::read_dir(&self.directory).chain_err(|| "failed reading asset dir")? {
|
||||
let entry = entry.chain_err(|| "invalid fh")?;
|
||||
let filetype = entry.file_type().chain_err(|| "failed getting file type")?;
|
||||
if !filetype.is_dir() || entry.file_name().len() != DIR_PARTITION_LEN {
|
||||
continue;
|
||||
}
|
||||
|
||||
for file_entry in
|
||||
fs::read_dir(entry.path()).chain_err(|| "failed reading asset subdir")?
|
||||
{
|
||||
let file_entry = file_entry.chain_err(|| "invalid fh")?;
|
||||
let path = file_entry.path();
|
||||
if path.extension().and_then(|e| e.to_str()) != Some("json") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let asset_id = AssetId::from_str(
|
||||
path.file_stem()
|
||||
.unwrap() // cannot fail if extension() succeeded
|
||||
.to_str()
|
||||
.chain_err(|| "invalid filename")?,
|
||||
)
|
||||
.chain_err(|| "invalid filename")?;
|
||||
|
||||
let modified = file_entry
|
||||
.metadata()
|
||||
.chain_err(|| "failed reading metadata")?
|
||||
.modified()
|
||||
.chain_err(|| "metadata modified failed")?;
|
||||
|
||||
if let Some((last_update, _)) = self.assets_cache.get(&asset_id) {
|
||||
if *last_update == modified {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let metadata: AssetMeta = serde_json::from_str(
|
||||
&fs::read_to_string(path).chain_err(|| "failed reading file")?,
|
||||
)
|
||||
.chain_err(|| "failed parsing file")?;
|
||||
|
||||
self.assets_cache.insert(asset_id, (modified, metadata));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn spawn_sync(asset_db: Arc<RwLock<AssetRegistry>>) -> thread::JoinHandle<()> {
|
||||
crate::util::spawn_thread("asset-registry", move || loop {
|
||||
if let Err(e) = asset_db.write().unwrap().fs_sync() {
|
||||
error!("registry fs_sync failed: {:?}", e);
|
||||
}
|
||||
|
||||
thread::sleep(Duration::from_secs(15));
|
||||
// TODO handle shutdowm
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct AssetMeta {
|
||||
#[serde(skip_serializing_if = "JsonValue::is_null")]
|
||||
pub contract: JsonValue,
|
||||
#[serde(skip_serializing_if = "JsonValue::is_null")]
|
||||
pub entity: JsonValue,
|
||||
pub precision: u8,
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ticker: Option<String>,
|
||||
}
|
||||
|
||||
impl AssetMeta {
|
||||
pub(crate) fn domain(&self) -> Option<&str> {
|
||||
self.entity["domain"].as_str()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AssetSorting(AssetSortField, AssetSortDir);
|
||||
|
||||
pub enum AssetSortField {
|
||||
Name,
|
||||
Domain,
|
||||
Ticker,
|
||||
}
|
||||
pub enum AssetSortDir {
|
||||
Descending,
|
||||
Ascending,
|
||||
}
|
||||
|
||||
type Comparator = Box<dyn Fn(&AssetEntry, &AssetEntry) -> cmp::Ordering>;
|
||||
|
||||
impl AssetSorting {
|
||||
#[allow(clippy::wrong_self_convention)]
|
||||
fn as_comparator(self) -> Comparator {
|
||||
let sort_fn: Comparator = match self.0 {
|
||||
AssetSortField::Name => {
|
||||
// Order by name first, use asset id as a tie breaker. the other sorting fields
|
||||
// don't require this because they're guaranteed to be unique.
|
||||
Box::new(|a, b| lc_cmp(&a.1.name, &b.1.name).then_with(|| a.0.cmp(b.0)))
|
||||
}
|
||||
AssetSortField::Domain => Box::new(|a, b| a.1.domain().cmp(&b.1.domain())),
|
||||
AssetSortField::Ticker => Box::new(|a, b| lc_cmp_opt(&a.1.ticker, &b.1.ticker)),
|
||||
};
|
||||
|
||||
match self.1 {
|
||||
AssetSortDir::Ascending => sort_fn,
|
||||
AssetSortDir::Descending => Box::new(move |a, b| sort_fn(a, b).reverse()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_query_params(query: &HashMap<String, String>) -> Result<Self> {
|
||||
let field = match query.get("sort_field").map(String::as_str) {
|
||||
None => AssetSortField::Ticker,
|
||||
Some("name") => AssetSortField::Name,
|
||||
Some("domain") => AssetSortField::Domain,
|
||||
Some("ticker") => AssetSortField::Ticker,
|
||||
_ => bail!("invalid sort field"),
|
||||
};
|
||||
|
||||
let dir = match query.get("sort_dir").map(String::as_str) {
|
||||
None => AssetSortDir::Ascending,
|
||||
Some("asc") => AssetSortDir::Ascending,
|
||||
Some("desc") => AssetSortDir::Descending,
|
||||
_ => bail!("invalid sort direction"),
|
||||
};
|
||||
|
||||
Ok(Self(field, dir))
|
||||
}
|
||||
}
|
||||
|
||||
fn lc_cmp(a: &str, b: &str) -> cmp::Ordering {
|
||||
a.to_lowercase().cmp(&b.to_lowercase())
|
||||
}
|
||||
fn lc_cmp_opt(a: &Option<String>, b: &Option<String>) -> cmp::Ordering {
|
||||
a.as_ref()
|
||||
.map(|a| a.to_lowercase())
|
||||
.cmp(&b.as_ref().map(|b| b.to_lowercase()))
|
||||
}
|
||||
|
||||
fn search_by<'a, I, F>(
|
||||
candidates: I,
|
||||
query: &str,
|
||||
limit: usize,
|
||||
field: F,
|
||||
) -> (Vec<AssetEntry<'a>>, Vec<AssetEntry<'a>>)
|
||||
where
|
||||
I: IntoIterator<Item = AssetEntry<'a>>,
|
||||
F: Fn(&AssetMeta) -> Option<&str>,
|
||||
{
|
||||
let mut matches = vec![];
|
||||
let mut remaining = vec![];
|
||||
|
||||
for (asset_id, metadata) in candidates {
|
||||
let position = field(metadata).and_then(|field| {
|
||||
// registry fields are ascii, so we don't need full unicode case-folding
|
||||
ascii_ci_find(field, query).map(|position| (position, field))
|
||||
});
|
||||
|
||||
if let Some((position, field)) = position {
|
||||
if matches.len() >= SEARCH_SORT_CANDIDATE_LIMIT {
|
||||
continue;
|
||||
}
|
||||
matches.push((position, field, asset_id, metadata));
|
||||
} else {
|
||||
remaining.push((asset_id, metadata));
|
||||
}
|
||||
}
|
||||
|
||||
matches.sort_unstable_by(|a, b| {
|
||||
a.0.cmp(&b.0)
|
||||
.then_with(|| ascii_ci_cmp(a.1, b.1))
|
||||
.then_with(|| a.2.cmp(b.2))
|
||||
});
|
||||
|
||||
(
|
||||
matches
|
||||
.into_iter()
|
||||
.take(limit)
|
||||
.map(|(_, _, asset_id, metadata)| (asset_id, metadata))
|
||||
.collect(),
|
||||
remaining,
|
||||
)
|
||||
}
|
||||
|
||||
// zero-allocation case-insensitive ASCII substring search
|
||||
// returns the byte offset of the first match
|
||||
fn ascii_ci_find(haystack: &str, needle: &str) -> Option<usize> {
|
||||
let (haystack, needle) = (haystack.as_bytes(), needle.as_bytes());
|
||||
if needle.is_empty() {
|
||||
return Some(0);
|
||||
}
|
||||
haystack
|
||||
.windows(needle.len())
|
||||
.position(|window| window.eq_ignore_ascii_case(needle))
|
||||
}
|
||||
|
||||
// zero-allocation case-insensitive ASCII string comparison
|
||||
fn ascii_ci_cmp(a: &str, b: &str) -> cmp::Ordering {
|
||||
let (a, b) = (a.as_bytes(), b.as_bytes());
|
||||
for i in 0..a.len().min(b.len()) {
|
||||
match a[i].to_ascii_lowercase().cmp(&b[i].to_ascii_lowercase()) {
|
||||
cmp::Ordering::Equal => continue,
|
||||
ord => return ord,
|
||||
}
|
||||
}
|
||||
a.len().cmp(&b.len())
|
||||
}
|
||||
@ -1,4 +1,5 @@
|
||||
#![allow(unexpected_cfgs)]
|
||||
use chan_signal::Signal;
|
||||
|
||||
error_chain! {
|
||||
types {
|
||||
Error, ErrorKind, ResultExt, Result;
|
||||
@ -10,33 +11,9 @@ error_chain! {
|
||||
display("Connection error: {}", msg)
|
||||
}
|
||||
|
||||
Interrupt(sig: i32) {
|
||||
Interrupt(signal: Signal) {
|
||||
description("Interruption by external signal")
|
||||
display("Iterrupted by signal {}", sig)
|
||||
display("Iterrupted by SIG{:?}", signal)
|
||||
}
|
||||
|
||||
TooManyUtxos(limit: usize) {
|
||||
description("Too many unspent transaction outputs. Contact support to raise limits.")
|
||||
display("Too many unspent transaction outputs (>{}). Contact support to raise limits.", limit)
|
||||
}
|
||||
|
||||
TooManyTxs(limit: usize) {
|
||||
description("Too many history transactions. Contact support to raise limits.")
|
||||
display("Too many history transactions (>{}). Contact support to raise limits.", limit)
|
||||
}
|
||||
|
||||
#[cfg(feature = "electrum-discovery")]
|
||||
ElectrumClient(e: electrum_client::Error) {
|
||||
description("Electrum client error")
|
||||
display("Electrum client error: {:?}", e)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "electrum-discovery")]
|
||||
impl From<electrum_client::Error> for Error {
|
||||
fn from(e: electrum_client::Error) -> Self {
|
||||
Error::from(ErrorKind::ElectrumClient(e))
|
||||
}
|
||||
}
|
||||
|
||||
38
src/lib.rs
38
src/lib.rs
@ -1,7 +1,38 @@
|
||||
#![recursion_limit = "1024"]
|
||||
|
||||
extern crate base64;
|
||||
extern crate bincode;
|
||||
extern crate bitcoin;
|
||||
extern crate bitcoin_bech32;
|
||||
extern crate bitcoin_hashes;
|
||||
extern crate chan_signal;
|
||||
extern crate crypto;
|
||||
extern crate dirs;
|
||||
extern crate glob;
|
||||
extern crate hex;
|
||||
extern crate hyper;
|
||||
extern crate itertools;
|
||||
extern crate libc;
|
||||
extern crate lru;
|
||||
extern crate lru_cache;
|
||||
extern crate num_cpus;
|
||||
extern crate page_size;
|
||||
extern crate prometheus;
|
||||
extern crate rayon;
|
||||
extern crate rocksdb;
|
||||
extern crate secp256k1;
|
||||
extern crate serde;
|
||||
extern crate stderrlog;
|
||||
extern crate sysconf;
|
||||
extern crate time;
|
||||
extern crate tiny_http;
|
||||
extern crate url;
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
extern crate elements;
|
||||
|
||||
#[macro_use]
|
||||
extern crate chan;
|
||||
#[macro_use]
|
||||
extern crate clap;
|
||||
#[macro_use]
|
||||
@ -15,19 +46,14 @@ extern crate serde_derive;
|
||||
#[macro_use]
|
||||
extern crate serde_json;
|
||||
|
||||
#[macro_use]
|
||||
extern crate lazy_static;
|
||||
|
||||
pub mod chain;
|
||||
pub mod config;
|
||||
pub mod daemon;
|
||||
pub mod electrum;
|
||||
pub mod errors;
|
||||
pub mod metrics;
|
||||
pub mod new_index;
|
||||
pub mod rest;
|
||||
pub mod signal;
|
||||
pub mod util;
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub mod elements;
|
||||
pub mod new_index;
|
||||
|
||||
@ -5,6 +5,7 @@ use std::io;
|
||||
use std::net::SocketAddr;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
use sysconf;
|
||||
use tiny_http;
|
||||
|
||||
pub use prometheus::{
|
||||
@ -66,9 +67,11 @@ impl Metrics {
|
||||
}
|
||||
|
||||
pub fn start(&self) {
|
||||
let server = tiny_http::Server::http(self.addr)
|
||||
.unwrap_or_else(|_| panic!("failed to start monitoring HTTP server at {}", self.addr));
|
||||
start_process_exporter(self);
|
||||
let server = tiny_http::Server::http(self.addr).expect(&format!(
|
||||
"failed to start monitoring HTTP server at {}",
|
||||
self.addr
|
||||
));
|
||||
start_process_exporter(&self);
|
||||
let reg = self.reg.clone();
|
||||
spawn_thread("metrics", move || loop {
|
||||
if let Err(e) = handle_request(®, server.recv()) {
|
||||
@ -97,14 +100,6 @@ struct Stats {
|
||||
fds: usize,
|
||||
}
|
||||
|
||||
fn get_ticks_per_second() -> Result<f64> {
|
||||
// Safety: This code is taken directly from sysconf
|
||||
match unsafe { libc::sysconf(libc::_SC_CLK_TCK) } {
|
||||
-1 => Err("Clock Tick unsupported".into()),
|
||||
ret => Ok(ret as f64),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_stats() -> Result<Stats> {
|
||||
if cfg!(target_os = "macos") {
|
||||
return Ok(Stats {
|
||||
@ -116,14 +111,15 @@ fn parse_stats() -> Result<Stats> {
|
||||
let value = fs::read_to_string("/proc/self/stat").chain_err(|| "failed to read stats")?;
|
||||
let parts: Vec<&str> = value.split_whitespace().collect();
|
||||
let page_size = page_size::get() as u64;
|
||||
let ticks_per_second = get_ticks_per_second().expect("failed to get _SC_CLK_TCK");
|
||||
let ticks_per_second = sysconf::raw::sysconf(sysconf::raw::SysconfVariable::ScClkTck)
|
||||
.expect("failed to get _SC_CLK_TCK") as f64;
|
||||
|
||||
let parse_part = |index: usize, name: &str| -> Result<u64> {
|
||||
parts
|
||||
Ok(parts
|
||||
.get(index)
|
||||
.chain_err(|| format!("missing {}: {:?}", name, parts))?
|
||||
.parse::<u64>()
|
||||
.chain_err(|| format!("invalid {}: {:?}", name, parts))
|
||||
.chain_err(|| format!("invalid {}: {:?}", name, parts))?)
|
||||
};
|
||||
|
||||
// For details, see '/proc/[pid]/stat' section at `man 5 proc`:
|
||||
@ -148,7 +144,7 @@ fn start_process_exporter(metrics: &Metrics) {
|
||||
spawn_thread("exporter", move || loop {
|
||||
match parse_stats() {
|
||||
Ok(stats) => {
|
||||
cpu.with_label_values(&["utime"]).set(stats.utime);
|
||||
cpu.with_label_values(&["utime"]).set(stats.utime as f64);
|
||||
rss.set(stats.rss as i64);
|
||||
fds.set(stats.fds as i64);
|
||||
}
|
||||
|
||||
@ -2,14 +2,9 @@ use rocksdb;
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::util::{bincode_util, Bytes};
|
||||
use crate::util::Bytes;
|
||||
|
||||
/// Each version will break any running instance with a DB that has a differing version.
|
||||
/// It will also break if light mode is enabled or disabled.
|
||||
// 1 = Original DB (since fork from Blockstream)
|
||||
// 2 = Add tx position to TxHistory rows and place Spending before Funding
|
||||
static DB_VERSION: u32 = 2;
|
||||
static DB_VERSION: u32 = 1;
|
||||
|
||||
#[derive(Debug, Eq, PartialEq)]
|
||||
pub struct DBRow {
|
||||
@ -23,14 +18,14 @@ pub struct ScanIterator<'a> {
|
||||
done: bool,
|
||||
}
|
||||
|
||||
impl Iterator for ScanIterator<'_> {
|
||||
impl<'a> Iterator for ScanIterator<'a> {
|
||||
type Item = DBRow;
|
||||
|
||||
fn next(&mut self) -> Option<DBRow> {
|
||||
if self.done {
|
||||
return None;
|
||||
}
|
||||
let (key, value) = self.iter.next().map(Result::ok)??;
|
||||
let (key, value) = self.iter.next()?;
|
||||
if !key.starts_with(&self.prefix) {
|
||||
self.done = true;
|
||||
return None;
|
||||
@ -48,7 +43,7 @@ pub struct ReverseScanIterator<'a> {
|
||||
done: bool,
|
||||
}
|
||||
|
||||
impl Iterator for ReverseScanIterator<'_> {
|
||||
impl<'a> Iterator for ReverseScanIterator<'a> {
|
||||
type Item = DBRow;
|
||||
|
||||
fn next(&mut self) -> Option<DBRow> {
|
||||
@ -61,76 +56,11 @@ impl Iterator for ReverseScanIterator<'_> {
|
||||
self.done = true;
|
||||
return None;
|
||||
}
|
||||
|
||||
let row = DBRow {
|
||||
key: key.into(),
|
||||
value: self.iter.value().unwrap().into(),
|
||||
};
|
||||
let value = self.iter.value().unwrap();
|
||||
|
||||
self.iter.prev();
|
||||
|
||||
Some(row)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ReverseScanGroupIterator<'a> {
|
||||
iters: Vec<ReverseScanIterator<'a>>,
|
||||
next_rows: Vec<Option<DBRow>>,
|
||||
value_offset: usize,
|
||||
done: bool,
|
||||
}
|
||||
|
||||
impl<'a> ReverseScanGroupIterator<'a> {
|
||||
pub fn new(
|
||||
mut iters: Vec<ReverseScanIterator<'a>>,
|
||||
value_offset: usize,
|
||||
) -> ReverseScanGroupIterator<'a> {
|
||||
let mut next_rows: Vec<Option<DBRow>> = Vec::with_capacity(iters.len());
|
||||
for iter in &mut iters {
|
||||
let next = iter.next();
|
||||
next_rows.push(next);
|
||||
}
|
||||
let done = next_rows.iter().all(|row| row.is_none());
|
||||
ReverseScanGroupIterator {
|
||||
iters,
|
||||
next_rows,
|
||||
value_offset,
|
||||
done,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for ReverseScanGroupIterator<'_> {
|
||||
type Item = DBRow;
|
||||
|
||||
fn next(&mut self) -> Option<DBRow> {
|
||||
if self.done {
|
||||
return None;
|
||||
}
|
||||
|
||||
let best_index = self
|
||||
.next_rows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by(|(a_index, a_opt), (b_index, b_opt)| match (a_opt, b_opt) {
|
||||
(None, None) => a_index.cmp(b_index),
|
||||
|
||||
(Some(_), None) => std::cmp::Ordering::Greater,
|
||||
|
||||
(None, Some(_)) => std::cmp::Ordering::Less,
|
||||
|
||||
(Some(a), Some(b)) => a.key[self.value_offset..].cmp(&(b.key[self.value_offset..])),
|
||||
})
|
||||
.map(|(index, _)| index)
|
||||
.unwrap_or(0);
|
||||
|
||||
let best = self.next_rows[best_index].take();
|
||||
self.next_rows[best_index] = self.iters.get_mut(best_index)?.next();
|
||||
if self.next_rows.iter().all(|row| row.is_none()) {
|
||||
self.done = true;
|
||||
}
|
||||
|
||||
best
|
||||
Some(DBRow { key, value })
|
||||
}
|
||||
}
|
||||
|
||||
@ -146,11 +76,28 @@ pub enum DBFlush {
|
||||
}
|
||||
|
||||
impl DB {
|
||||
pub fn open(path: &Path, config: &Config) -> DB {
|
||||
pub fn open(path: &Path) -> DB {
|
||||
debug!("opening DB at {:?}", path);
|
||||
let mut db_opts = rocksdb::Options::default();
|
||||
db_opts.create_if_missing(true);
|
||||
db_opts.set_max_open_files(-1); // TODO: make sure to `ulimit -n` this process correctly
|
||||
db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
|
||||
db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy);
|
||||
db_opts.set_target_file_size_base(256 << 20);
|
||||
db_opts.set_write_buffer_size(256 << 20);
|
||||
db_opts.set_disable_auto_compactions(true); // for initial bulk load
|
||||
|
||||
// db_opts.set_advise_random_on_open(???);
|
||||
db_opts.set_compaction_readahead_size(1 << 20);
|
||||
db_opts.increase_parallelism(2);
|
||||
|
||||
// let mut block_opts = rocksdb::BlockBasedOptions::default();
|
||||
// block_opts.set_block_size(???);
|
||||
|
||||
let db = DB {
|
||||
db: open_raw_db(path, OpenMode::ReadWrite),
|
||||
db: rocksdb::DB::open(&db_opts, path).expect("failed to open RocksDB"),
|
||||
};
|
||||
db.verify_compatibility(config);
|
||||
db.verify_compatibility();
|
||||
db
|
||||
}
|
||||
|
||||
@ -166,11 +113,11 @@ impl DB {
|
||||
self.db.set_options(&opts).unwrap();
|
||||
}
|
||||
|
||||
pub fn raw_iterator(&self) -> rocksdb::DBRawIterator<'_> {
|
||||
pub fn raw_iterator(&self) -> rocksdb::DBRawIterator {
|
||||
self.db.raw_iterator()
|
||||
}
|
||||
|
||||
pub fn iter_scan(&self, prefix: &[u8]) -> ScanIterator<'_> {
|
||||
pub fn iter_scan(&self, prefix: &[u8]) -> ScanIterator {
|
||||
ScanIterator {
|
||||
prefix: prefix.to_vec(),
|
||||
iter: self.db.prefix_iterator(prefix),
|
||||
@ -178,7 +125,7 @@ impl DB {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn iter_scan_from(&self, prefix: &[u8], start_at: &[u8]) -> ScanIterator<'_> {
|
||||
pub fn iter_scan_from(&self, prefix: &[u8], start_at: &[u8]) -> ScanIterator {
|
||||
let iter = self.db.iterator(rocksdb::IteratorMode::From(
|
||||
start_at,
|
||||
rocksdb::Direction::Forward,
|
||||
@ -190,7 +137,7 @@ impl DB {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn iter_scan_reverse(&self, prefix: &[u8], prefix_max: &[u8]) -> ReverseScanIterator<'_> {
|
||||
pub fn iter_scan_reverse(&self, prefix: &[u8], prefix_max: &[u8]) -> ReverseScanIterator {
|
||||
let mut iter = self.db.raw_iterator();
|
||||
iter.seek_for_prev(prefix_max);
|
||||
|
||||
@ -201,25 +148,6 @@ impl DB {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn iter_scan_group_reverse(
|
||||
&self,
|
||||
prefixes: impl Iterator<Item = (Vec<u8>, Vec<u8>)>,
|
||||
value_offset: usize,
|
||||
) -> ReverseScanGroupIterator<'_> {
|
||||
let iters = prefixes
|
||||
.map(|(prefix, prefix_max)| {
|
||||
let mut iter = self.db.raw_iterator();
|
||||
iter.seek_for_prev(prefix_max);
|
||||
ReverseScanIterator {
|
||||
prefix: prefix.to_vec(),
|
||||
iter,
|
||||
done: false,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
ReverseScanGroupIterator::new(iters, value_offset)
|
||||
}
|
||||
|
||||
pub fn write(&self, mut rows: Vec<DBRow>, flush: DBFlush) {
|
||||
debug!(
|
||||
"writing {} rows to {:?}, flush={:?}",
|
||||
@ -230,7 +158,7 @@ impl DB {
|
||||
rows.sort_unstable_by(|a, b| a.key.cmp(&b.key));
|
||||
let mut batch = rocksdb::WriteBatch::default();
|
||||
for row in rows {
|
||||
batch.put(&row.key, &row.value);
|
||||
batch.put(&row.key, &row.value).unwrap();
|
||||
}
|
||||
let do_flush = match flush {
|
||||
DBFlush::Enable => true,
|
||||
@ -242,43 +170,16 @@ impl DB {
|
||||
self.db.write_opt(batch, &opts).unwrap();
|
||||
}
|
||||
|
||||
pub fn delete(&self, keys: Vec<Vec<u8>>) {
|
||||
debug!("deleting {} rows from {:?}", keys.len(), self.db);
|
||||
for key in keys {
|
||||
let _ = self.db.delete(key).inspect_err(|err| {
|
||||
warn!("Error while deleting DB row: {err}");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub fn flush(&self) {
|
||||
self.db.flush().unwrap();
|
||||
}
|
||||
|
||||
pub fn put(&self, key: &[u8], value: &[u8]) {
|
||||
self.db.put(key, value).unwrap();
|
||||
}
|
||||
|
||||
pub fn put_sync(&self, key: &[u8], value: &[u8]) {
|
||||
let mut opts = rocksdb::WriteOptions::new();
|
||||
opts.set_sync(true);
|
||||
self.db.put_opt(key, value, &opts).unwrap();
|
||||
}
|
||||
|
||||
pub fn get(&self, key: &[u8]) -> Option<Bytes> {
|
||||
self.db.get(key).unwrap().map(|v| v.to_vec())
|
||||
}
|
||||
|
||||
fn verify_compatibility(&self, config: &Config) {
|
||||
let mut compatibility_bytes = bincode_util::serialize_little(&DB_VERSION).unwrap();
|
||||
|
||||
if config.light_mode {
|
||||
// append a byte to indicate light_mode is enabled.
|
||||
// we're not letting bincode serialize this so that the compatiblity bytes won't change
|
||||
// (and require a reindex) when light_mode is disabled. this should be chagned the next
|
||||
// time we bump DB_VERSION and require a re-index anyway.
|
||||
compatibility_bytes.push(1);
|
||||
}
|
||||
fn verify_compatibility(&self) {
|
||||
let compatibility_bytes = bincode::serialize(&DB_VERSION).unwrap();
|
||||
|
||||
match self.get(b"V") {
|
||||
None => self.put(b"V", &compatibility_bytes),
|
||||
@ -290,41 +191,34 @@ impl DB {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
#[repr(u8)]
|
||||
pub enum OpenMode {
|
||||
ReadOnly,
|
||||
ReadWrite,
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
pub fn open_raw_db<T: rocksdb::ThreadMode>(
|
||||
path: &Path,
|
||||
read_mode: OpenMode,
|
||||
) -> rocksdb::DBWithThreadMode<T> {
|
||||
debug!("opening DB at {:?}", path);
|
||||
let mut db_opts = rocksdb::Options::default();
|
||||
db_opts.create_if_missing(true);
|
||||
db_opts.set_max_open_files(100_000); // TODO: make sure to `ulimit -n` this process correctly
|
||||
db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
|
||||
db_opts.set_compression_type(rocksdb::DBCompressionType::None);
|
||||
db_opts.set_target_file_size_base(1_073_741_824);
|
||||
db_opts.set_write_buffer_size(256 << 20);
|
||||
db_opts.set_disable_auto_compactions(true); // for initial bulk load
|
||||
use crate::new_index::db::{DBRow, DB};
|
||||
use tempfile;
|
||||
|
||||
// db_opts.set_advise_random_on_open(???);
|
||||
db_opts.set_compaction_readahead_size(1 << 20);
|
||||
db_opts.increase_parallelism(2);
|
||||
|
||||
// let mut block_opts = rocksdb::BlockBasedOptions::default();
|
||||
// block_opts.set_block_size(???);
|
||||
|
||||
match read_mode {
|
||||
OpenMode::ReadOnly => {
|
||||
rocksdb::DBWithThreadMode::<T>::open_for_read_only(&db_opts, path, false)
|
||||
.expect("failed to open RocksDB (READ ONLY)")
|
||||
}
|
||||
OpenMode::ReadWrite => {
|
||||
rocksdb::DBWithThreadMode::<T>::open(&db_opts, path).expect("failed to open RocksDB")
|
||||
#[test]
|
||||
fn test_db_iterator() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = DB::open(dir.path());
|
||||
let rows = vec![
|
||||
DBRow {
|
||||
key: b"X11".to_vec(),
|
||||
value: b"a".to_vec(),
|
||||
},
|
||||
DBRow {
|
||||
key: b"X22".to_vec(),
|
||||
value: b"b".to_vec(),
|
||||
},
|
||||
DBRow {
|
||||
key: b"X33".to_vec(),
|
||||
value: b"c".to_vec(),
|
||||
},
|
||||
];
|
||||
for row in &rows {
|
||||
db.put(&row.key, &row.value);
|
||||
}
|
||||
let actual: Vec<DBRow> = db.iter_scan(b"X").collect();
|
||||
assert_eq!(rows, actual);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,17 +1,17 @@
|
||||
use rayon::prelude::*;
|
||||
use crate::chain::Block;
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
use bitcoin::consensus::encode::{deserialize, Decodable};
|
||||
#[cfg(feature = "liquid")]
|
||||
use elements::encode::{deserialize, Decodable};
|
||||
use bitcoin::consensus::encode::{deserialize, serialize, Decodable};
|
||||
use bitcoin::util::hash::BitcoinHash;
|
||||
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::io::Cursor;
|
||||
use std::io::{Cursor, Seek, SeekFrom};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::mpsc::Receiver;
|
||||
use std::thread;
|
||||
|
||||
use crate::chain::{Block, BlockHash, BlockSizeCompat};
|
||||
use crate::daemon::Daemon;
|
||||
use crate::errors::*;
|
||||
use crate::util::{spawn_thread, HeaderEntry, SyncChannel};
|
||||
@ -42,70 +42,19 @@ pub struct BlockEntry {
|
||||
|
||||
type SizedBlock = (Block, u32);
|
||||
|
||||
pub struct SequentialFetcher<T> {
|
||||
fetcher: Box<dyn FnOnce() -> Vec<Vec<T>>>,
|
||||
}
|
||||
|
||||
impl<T> SequentialFetcher<T> {
|
||||
fn from<F: FnOnce() -> Vec<Vec<T>> + 'static>(pre_func: F) -> Self {
|
||||
SequentialFetcher {
|
||||
fetcher: Box::new(pre_func),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn map<FN>(self, mut func: FN)
|
||||
where
|
||||
FN: FnMut(Vec<T>),
|
||||
{
|
||||
for item in (self.fetcher)() {
|
||||
func(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bitcoind_sequential_fetcher(
|
||||
daemon: &Daemon,
|
||||
new_headers: Vec<HeaderEntry>,
|
||||
) -> Result<SequentialFetcher<BlockEntry>> {
|
||||
let daemon = daemon.reconnect()?;
|
||||
Ok(SequentialFetcher::from(move || {
|
||||
new_headers
|
||||
.chunks(100)
|
||||
.map(|entries| {
|
||||
let blockhashes: Vec<BlockHash> = entries.iter().map(|e| *e.hash()).collect();
|
||||
let blocks = daemon
|
||||
.getblocks(&blockhashes)
|
||||
.expect("failed to get blocks from bitcoind");
|
||||
assert_eq!(blocks.len(), entries.len());
|
||||
let block_entries: Vec<BlockEntry> = blocks
|
||||
.into_iter()
|
||||
.zip(entries)
|
||||
.map(|(block, entry)| BlockEntry {
|
||||
entry: entry.clone(), // TODO: remove this clone()
|
||||
size: block.get_block_size() as u32,
|
||||
block,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(block_entries.len(), entries.len());
|
||||
block_entries
|
||||
})
|
||||
.collect()
|
||||
}))
|
||||
}
|
||||
|
||||
pub struct Fetcher<T> {
|
||||
receiver: crossbeam_channel::Receiver<T>,
|
||||
receiver: Receiver<T>,
|
||||
thread: thread::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl<T> Fetcher<T> {
|
||||
fn from(receiver: crossbeam_channel::Receiver<T>, thread: thread::JoinHandle<()>) -> Self {
|
||||
fn from(receiver: Receiver<T>, thread: thread::JoinHandle<()>) -> Self {
|
||||
Fetcher { receiver, thread }
|
||||
}
|
||||
|
||||
pub fn map<F>(self, mut func: F)
|
||||
where
|
||||
F: FnMut(T),
|
||||
F: FnMut(T) -> (),
|
||||
{
|
||||
for item in self.receiver {
|
||||
func(item);
|
||||
@ -118,17 +67,17 @@ fn bitcoind_fetcher(
|
||||
daemon: &Daemon,
|
||||
new_headers: Vec<HeaderEntry>,
|
||||
) -> Result<Fetcher<Vec<BlockEntry>>> {
|
||||
if let Some(tip) = new_headers.last() {
|
||||
debug!("{:?} ({} left to index)", tip, new_headers.len());
|
||||
};
|
||||
new_headers.last().map(|tip| {
|
||||
debug!("{:?} ({} new blocks to fetch)", tip, new_headers.len());
|
||||
});
|
||||
let daemon = daemon.reconnect()?;
|
||||
let chan = SyncChannel::new(1);
|
||||
let sender = chan.sender();
|
||||
Ok(Fetcher::from(
|
||||
chan.into_receiver(),
|
||||
spawn_thread("bitcoind_fetcher", move || {
|
||||
spawn_thread("bitcoind_fetcher", move || -> () {
|
||||
for entries in new_headers.chunks(100) {
|
||||
let blockhashes: Vec<BlockHash> = entries.iter().map(|e| *e.hash()).collect();
|
||||
let blockhashes: Vec<Sha256dHash> = entries.iter().map(|e| *e.hash()).collect();
|
||||
let blocks = daemon
|
||||
.getblocks(&blockhashes)
|
||||
.expect("failed to get blocks from bitcoind");
|
||||
@ -137,8 +86,8 @@ fn bitcoind_fetcher(
|
||||
.into_iter()
|
||||
.zip(entries)
|
||||
.map(|(block, entry)| BlockEntry {
|
||||
entry: entry.clone(), // TODO: remove this clone()
|
||||
size: block.get_block_size() as u32,
|
||||
entry: entry.clone(), // TODO: remove this clone()
|
||||
size: serialize(&block).len() as u32, // TODO: avoid re-serializing
|
||||
block,
|
||||
})
|
||||
.collect();
|
||||
@ -161,18 +110,18 @@ fn blkfiles_fetcher(
|
||||
let chan = SyncChannel::new(1);
|
||||
let sender = chan.sender();
|
||||
|
||||
let mut entry_map: HashMap<BlockHash, HeaderEntry> =
|
||||
let mut entry_map: HashMap<Sha256dHash, HeaderEntry> =
|
||||
new_headers.into_iter().map(|h| (*h.hash(), h)).collect();
|
||||
|
||||
let parser = blkfiles_parser(blkfiles_reader(blk_files), magic);
|
||||
Ok(Fetcher::from(
|
||||
chan.into_receiver(),
|
||||
spawn_thread("blkfiles_fetcher", move || {
|
||||
spawn_thread("blkfiles_fetcher", move || -> () {
|
||||
parser.map(|sizedblocks| {
|
||||
let block_entries: Vec<BlockEntry> = sizedblocks
|
||||
.into_iter()
|
||||
.filter_map(|(block, size)| {
|
||||
let blockhash = block.block_hash();
|
||||
let blockhash = block.bitcoin_hash();
|
||||
entry_map
|
||||
.remove(&blockhash)
|
||||
.map(|entry| BlockEntry { block, entry, size })
|
||||
@ -200,36 +149,16 @@ fn blkfiles_fetcher(
|
||||
fn blkfiles_reader(blk_files: Vec<PathBuf>) -> Fetcher<Vec<u8>> {
|
||||
let chan = SyncChannel::new(1);
|
||||
let sender = chan.sender();
|
||||
let xor_key = blk_files.first().and_then(|p| {
|
||||
let xor_file = p
|
||||
.parent()
|
||||
.expect("blk.dat files must exist in a directory")
|
||||
.join("xor.dat");
|
||||
if xor_file.exists() {
|
||||
Some(fs::read(xor_file).expect("xor.dat exists"))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
Fetcher::from(
|
||||
chan.into_receiver(),
|
||||
spawn_thread("blkfiles_reader", move || {
|
||||
spawn_thread("blkfiles_reader", move || -> () {
|
||||
for path in blk_files {
|
||||
trace!("reading {:?}", path);
|
||||
let mut blob = fs::read(&path)
|
||||
.unwrap_or_else(|e| panic!("failed to read {:?}: {:?}", path, e));
|
||||
|
||||
// If the xor.dat exists. Use it to decrypt the block files.
|
||||
if let Some(xor_key) = &xor_key {
|
||||
for (&key, byte) in xor_key.iter().cycle().zip(blob.iter_mut()) {
|
||||
*byte ^= key;
|
||||
}
|
||||
}
|
||||
|
||||
let blob = fs::read(&path).expect(&format!("failed to read {:?}", path));
|
||||
sender
|
||||
.send(blob)
|
||||
.unwrap_or_else(|_| panic!("failed to send {:?} contents", path));
|
||||
.expect(&format!("failed to send {:?} contents", path));
|
||||
}
|
||||
}),
|
||||
)
|
||||
@ -241,7 +170,7 @@ fn blkfiles_parser(blobs: Fetcher<Vec<u8>>, magic: u32) -> Fetcher<Vec<SizedBloc
|
||||
|
||||
Fetcher::from(
|
||||
chan.into_receiver(),
|
||||
spawn_thread("blkfiles_parser", move || {
|
||||
spawn_thread("blkfiles_parser", move || -> () {
|
||||
blobs.map(|blob| {
|
||||
trace!("parsing {} bytes", blob.len());
|
||||
let blocks = parse_blocks(blob, magic).expect("failed to parse blk*.dat file");
|
||||
@ -257,40 +186,34 @@ fn parse_blocks(blob: Vec<u8>, magic: u32) -> Result<Vec<SizedBlock>> {
|
||||
let mut cursor = Cursor::new(&blob);
|
||||
let mut slices = vec![];
|
||||
let max_pos = blob.len() as u64;
|
||||
|
||||
while cursor.position() < max_pos {
|
||||
let offset = cursor.position();
|
||||
match u32::consensus_decode(&mut cursor) {
|
||||
Ok(value) => {
|
||||
if magic != value {
|
||||
cursor.set_position(offset + 1);
|
||||
cursor
|
||||
.seek(SeekFrom::Current(-3))
|
||||
.expect("failed to seek back");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
Err(_) => break, // EOF
|
||||
};
|
||||
let block_size = u32::consensus_decode(&mut cursor).chain_err(|| "no block size")?;
|
||||
let start = cursor.position();
|
||||
let end = start + block_size as u64;
|
||||
let start = cursor.position() as usize;
|
||||
cursor
|
||||
.seek(SeekFrom::Current(block_size as i64))
|
||||
.chain_err(|| format!("seek {} failed", block_size))?;
|
||||
let end = cursor.position() as usize;
|
||||
|
||||
// If Core's WriteBlockToDisk ftell fails, only the magic bytes and size will be written
|
||||
// and the block body won't be written to the blk*.dat file.
|
||||
// Since the first 4 bytes should contain the block's version, we can skip such blocks
|
||||
// by peeking the cursor (and skipping previous `magic` and `block_size`).
|
||||
match u32::consensus_decode(&mut cursor) {
|
||||
Ok(value) => {
|
||||
if magic == value {
|
||||
cursor.set_position(start);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
Err(_) => break, // EOF
|
||||
}
|
||||
slices.push((&blob[start as usize..end as usize], block_size));
|
||||
cursor.set_position(end);
|
||||
slices.push((&blob[start..end], block_size));
|
||||
}
|
||||
|
||||
Ok(super::THREAD_POOL.install(|| {
|
||||
let pool = rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(0) // CPU-bound
|
||||
.thread_name(|i| format!("parse-blocks-{}", i))
|
||||
.build()
|
||||
.unwrap();
|
||||
Ok(pool.install(|| {
|
||||
slices
|
||||
.into_par_iter()
|
||||
.map(|(slice, size)| (deserialize(slice).expect("failed to parse Block"), size))
|
||||
|
||||
@ -1,58 +1,46 @@
|
||||
use bounded_vec_deque::BoundedVecDeque;
|
||||
use arraydeque::{ArrayDeque, Wrapping};
|
||||
use bitcoin::consensus::encode::serialize;
|
||||
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
|
||||
use itertools::Itertools;
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
use bitcoin::consensus::encode::serialize;
|
||||
#[cfg(feature = "liquid")]
|
||||
use elements::{encode::serialize, AssetId};
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||
use std::collections::{BTreeSet, HashMap, HashSet};
|
||||
use std::iter::FromIterator;
|
||||
use std::ops::Bound::{Excluded, Unbounded};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::chain::{deserialize, Network, OutPoint, Transaction, TxOut, Txid, TxidCompat};
|
||||
use crate::config::Config;
|
||||
use crate::chain::{OutPoint, Transaction, TxOut};
|
||||
use crate::daemon::Daemon;
|
||||
use crate::errors::*;
|
||||
use crate::metrics::{GaugeVec, HistogramOpts, HistogramVec, MetricOpts, Metrics};
|
||||
use crate::new_index::{
|
||||
compute_script_hash, schema::FullHash, ChainQuery, FundingInfo, ScriptStats, SpendingInfo,
|
||||
SpendingInput, TxHistoryInfo, Utxo,
|
||||
compute_script_hash, parse_hash, schema::FullHash, ChainQuery, FundingInfo, ScriptStats,
|
||||
SpendingInfo, SpendingInput, TxHistoryInfo, Utxo,
|
||||
};
|
||||
use crate::util::fees::{make_fee_histogram, TxFeeInfo};
|
||||
use crate::util::{extract_tx_prevouts, full_hash, has_prevout, is_spendable, Bytes};
|
||||
use crate::util::{full_hash, has_prevout, is_spendable, Bytes};
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
use crate::elements::asset;
|
||||
const RECENT_TXS_SIZE: usize = 10;
|
||||
const BACKLOG_STATS_TTL: u64 = 10;
|
||||
|
||||
pub struct Mempool {
|
||||
chain: Arc<ChainQuery>,
|
||||
config: Arc<Config>,
|
||||
txstore: BTreeMap<Txid, Transaction>,
|
||||
feeinfo: HashMap<Txid, TxFeeInfo>,
|
||||
txstore: HashMap<Sha256dHash, Transaction>,
|
||||
feeinfo: HashMap<Sha256dHash, TxFeeInfo>,
|
||||
history: HashMap<FullHash, Vec<TxHistoryInfo>>, // ScriptHash -> {history_entries}
|
||||
edges: HashMap<OutPoint, (Txid, u32)>, // OutPoint -> (spending_txid, spending_vin)
|
||||
recent: BoundedVecDeque<TxOverview>, // The N most recent txs to enter the mempool
|
||||
edges: HashMap<OutPoint, (Sha256dHash, u32)>, // OutPoint -> (spending_txid, spending_vin)
|
||||
recent: ArrayDeque<[TxOverview; RECENT_TXS_SIZE], Wrapping>, // The N most recent txs to enter the mempool
|
||||
backlog_stats: (BacklogStats, Instant),
|
||||
|
||||
// monitoring
|
||||
latency: HistogramVec, // mempool requests latency
|
||||
delta: HistogramVec, // # of added/removed txs
|
||||
count: GaugeVec, // current state of the mempool
|
||||
|
||||
// elements only
|
||||
#[cfg(feature = "liquid")]
|
||||
pub asset_history: HashMap<AssetId, Vec<TxHistoryInfo>>,
|
||||
#[cfg(feature = "liquid")]
|
||||
pub asset_issuance: HashMap<AssetId, asset::AssetRow>,
|
||||
}
|
||||
|
||||
// A simplified transaction view used for the list of most recent transactions
|
||||
#[derive(Serialize)]
|
||||
pub struct TxOverview {
|
||||
txid: Txid,
|
||||
txid: Sha256dHash,
|
||||
fee: u64,
|
||||
vsize: u32,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
@ -60,17 +48,17 @@ pub struct TxOverview {
|
||||
}
|
||||
|
||||
impl Mempool {
|
||||
pub fn new(chain: Arc<ChainQuery>, metrics: &Metrics, config: Arc<Config>) -> Self {
|
||||
pub fn new(chain: Arc<ChainQuery>, metrics: &Metrics) -> Self {
|
||||
Mempool {
|
||||
chain,
|
||||
txstore: BTreeMap::new(),
|
||||
txstore: HashMap::new(),
|
||||
feeinfo: HashMap::new(),
|
||||
history: HashMap::new(),
|
||||
edges: HashMap::new(),
|
||||
recent: BoundedVecDeque::new(config.mempool_recent_txs_size),
|
||||
recent: ArrayDeque::new(),
|
||||
backlog_stats: (
|
||||
BacklogStats::default(),
|
||||
Instant::now() - Duration::from_secs(config.mempool_backlog_stats_ttl),
|
||||
Instant::now() - Duration::from_secs(BACKLOG_STATS_TTL),
|
||||
),
|
||||
latency: metrics.histogram_vec(
|
||||
HistogramOpts::new("mempool_latency", "Mempool requests latency (in seconds)"),
|
||||
@ -84,24 +72,14 @@ impl Mempool {
|
||||
MetricOpts::new("mempool_count", "# of elements currently at the mempool"),
|
||||
&["type"],
|
||||
),
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
asset_history: HashMap::new(),
|
||||
#[cfg(feature = "liquid")]
|
||||
asset_issuance: HashMap::new(),
|
||||
config,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn network(&self) -> Network {
|
||||
self.config.network_type
|
||||
pub fn lookup_txn(&self, txid: &Sha256dHash) -> Option<Transaction> {
|
||||
self.txstore.get(txid).map(|item| item.clone())
|
||||
}
|
||||
|
||||
pub fn lookup_txn(&self, txid: &Txid) -> Option<Transaction> {
|
||||
self.txstore.get(txid).cloned()
|
||||
}
|
||||
|
||||
pub fn lookup_raw_txn(&self, txid: &Txid) -> Option<Bytes> {
|
||||
pub fn lookup_raw_txn(&self, txid: &Sha256dHash) -> Option<Bytes> {
|
||||
self.txstore.get(txid).map(serialize)
|
||||
}
|
||||
|
||||
@ -117,122 +95,30 @@ impl Mempool {
|
||||
self.edges.contains_key(outpoint)
|
||||
}
|
||||
|
||||
pub fn get_tx_fee(&self, txid: &Txid) -> Option<u64> {
|
||||
Some(self.feeinfo.get(txid)?.fee)
|
||||
}
|
||||
|
||||
pub fn has_unconfirmed_parents(&self, txid: &Txid) -> bool {
|
||||
let tx = match self.txstore.get(txid) {
|
||||
Some(tx) => tx,
|
||||
None => return false,
|
||||
};
|
||||
tx.input
|
||||
.iter()
|
||||
.any(|txin| self.txstore.contains_key(&txin.previous_output.txid))
|
||||
}
|
||||
|
||||
pub fn history(
|
||||
&self,
|
||||
scripthash: &[u8],
|
||||
last_seen_txid: Option<&Txid>,
|
||||
limit: usize,
|
||||
) -> Vec<Transaction> {
|
||||
// TODO: return as Vec<(Transaction,Option<BlockId>)>?
|
||||
pub fn history(&self, scripthash: &[u8], limit: usize) -> Vec<Transaction> {
|
||||
let _timer = self.latency.with_label_values(&["history"]).start_timer();
|
||||
self.history
|
||||
.get(scripthash)
|
||||
.map_or_else(std::vec::Vec::new, |entries| {
|
||||
self._history(entries, last_seen_txid, limit)
|
||||
})
|
||||
match self.history.get(scripthash) {
|
||||
None => return vec![],
|
||||
Some(entries) => entries
|
||||
.iter()
|
||||
.map(get_entry_txid)
|
||||
.unique()
|
||||
.take(limit)
|
||||
.map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
|
||||
.cloned()
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn history_txids_iter<'a>(&'a self, scripthash: &[u8]) -> impl Iterator<Item = Txid> + 'a {
|
||||
self.history
|
||||
.get(scripthash)
|
||||
.into_iter()
|
||||
.flat_map(|v| v.iter().map(|e| e.get_txid()).unique())
|
||||
}
|
||||
|
||||
fn _history(
|
||||
&self,
|
||||
entries: &[TxHistoryInfo],
|
||||
last_seen_txid: Option<&Txid>,
|
||||
limit: usize,
|
||||
) -> Vec<Transaction> {
|
||||
entries
|
||||
.iter()
|
||||
.map(|e| e.get_txid())
|
||||
.unique()
|
||||
// TODO seek directly to last seen tx without reading earlier rows
|
||||
.skip_while(|txid| {
|
||||
// skip until we reach the last_seen_txid
|
||||
last_seen_txid.is_some_and(|last_seen_txid| last_seen_txid != txid)
|
||||
})
|
||||
.skip(match last_seen_txid {
|
||||
Some(_) => 1, // skip the last_seen_txid itself
|
||||
None => 0,
|
||||
})
|
||||
.take(limit)
|
||||
.map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn history_group(
|
||||
&self,
|
||||
scripthashes: &[[u8; 32]],
|
||||
last_seen_txid: Option<&Txid>,
|
||||
limit: usize,
|
||||
) -> Vec<Transaction> {
|
||||
let _timer = self
|
||||
.latency
|
||||
.with_label_values(&["history_group"])
|
||||
.start_timer();
|
||||
scripthashes
|
||||
.iter()
|
||||
.filter_map(|scripthash| self.history.get(&scripthash[..]))
|
||||
.flat_map(|entries| entries.iter())
|
||||
.map(|e| e.get_txid())
|
||||
.unique()
|
||||
// TODO seek directly to last seen tx without reading earlier rows
|
||||
.skip_while(|txid| {
|
||||
// skip until we reach the last_seen_txid
|
||||
last_seen_txid.is_some_and(|last_seen_txid| last_seen_txid != txid)
|
||||
})
|
||||
.skip(match last_seen_txid {
|
||||
Some(_) => 1, // skip the last_seen_txid itself
|
||||
None => 0,
|
||||
})
|
||||
.take(limit)
|
||||
.map(|txid| self.txstore.get(&txid).expect("missing mempool tx"))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn history_txids_iter_group<'a>(
|
||||
&'a self,
|
||||
scripthashes: &'a [[u8; 32]],
|
||||
) -> impl Iterator<Item = Txid> + 'a {
|
||||
scripthashes
|
||||
.iter()
|
||||
.filter_map(move |scripthash| self.history.get(&scripthash[..]))
|
||||
.flat_map(|entries| entries.iter())
|
||||
.map(|entry| entry.get_txid())
|
||||
.unique()
|
||||
}
|
||||
|
||||
pub fn history_txids(&self, scripthash: &[u8], limit: usize) -> Vec<Txid> {
|
||||
pub fn history_txids(&self, scripthash: &[u8]) -> Vec<Sha256dHash> {
|
||||
let _timer = self
|
||||
.latency
|
||||
.with_label_values(&["history_txids"])
|
||||
.start_timer();
|
||||
match self.history.get(scripthash) {
|
||||
None => vec![],
|
||||
Some(entries) => entries
|
||||
.iter()
|
||||
.map(|e| e.get_txid())
|
||||
.unique()
|
||||
.take(limit)
|
||||
.collect(),
|
||||
None => return vec![],
|
||||
Some(entries) => entries.iter().map(get_entry_txid).unique().collect(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -244,33 +130,15 @@ impl Mempool {
|
||||
};
|
||||
|
||||
entries
|
||||
.iter()
|
||||
.into_iter()
|
||||
.filter_map(|entry| match entry {
|
||||
TxHistoryInfo::Funding(info) => {
|
||||
// Liquid requires some additional information from the txo that's not available in the TxHistoryInfo index.
|
||||
#[cfg(feature = "liquid")]
|
||||
let txo = self.lookup_txo(&entry.get_funded_outpoint())?;
|
||||
|
||||
Some(Utxo {
|
||||
txid: deserialize(&info.txid).expect("invalid txid"),
|
||||
vout: info.vout,
|
||||
value: info.value,
|
||||
confirmed: None,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
asset: txo.asset,
|
||||
#[cfg(feature = "liquid")]
|
||||
nonce: txo.nonce,
|
||||
#[cfg(feature = "liquid")]
|
||||
witness: txo.witness,
|
||||
})
|
||||
}
|
||||
TxHistoryInfo::Spending(_) => None,
|
||||
#[cfg(feature = "liquid")]
|
||||
TxHistoryInfo::Issuing(_)
|
||||
| TxHistoryInfo::Burning(_)
|
||||
| TxHistoryInfo::Pegin(_)
|
||||
| TxHistoryInfo::Pegout(_) => unreachable!(),
|
||||
TxHistoryInfo::Funding(info) => Some(Utxo {
|
||||
txid: parse_hash(&info.txid),
|
||||
vout: info.vout as u32,
|
||||
value: info.value,
|
||||
confirmed: None,
|
||||
}),
|
||||
TxHistoryInfo::Spending(..) => None,
|
||||
})
|
||||
.filter(|utxo| !self.has_spend(&OutPoint::from(utxo)))
|
||||
.collect()
|
||||
@ -288,7 +156,7 @@ impl Mempool {
|
||||
};
|
||||
|
||||
for entry in entries {
|
||||
if seen_txids.insert(entry.get_txid()) {
|
||||
if seen_txids.insert(get_entry_txid(entry)) {
|
||||
stats.tx_count += 1;
|
||||
}
|
||||
|
||||
@ -298,27 +166,20 @@ impl Mempool {
|
||||
stats.funded_txo_count += 1;
|
||||
stats.funded_txo_sum += info.value;
|
||||
}
|
||||
#[cfg(feature = "liquid")]
|
||||
TxHistoryInfo::Funding(_) => {
|
||||
stats.funded_txo_count += 1;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
TxHistoryInfo::Spending(info) => {
|
||||
stats.spent_txo_count += 1;
|
||||
stats.spent_txo_sum += info.value;
|
||||
}
|
||||
|
||||
// Elements
|
||||
#[cfg(feature = "liquid")]
|
||||
TxHistoryInfo::Funding(_) => {
|
||||
stats.funded_txo_count += 1;
|
||||
}
|
||||
#[cfg(feature = "liquid")]
|
||||
TxHistoryInfo::Spending(_) => {
|
||||
stats.spent_txo_count += 1;
|
||||
}
|
||||
#[cfg(feature = "liquid")]
|
||||
TxHistoryInfo::Issuing(_)
|
||||
| TxHistoryInfo::Burning(_)
|
||||
| TxHistoryInfo::Pegin(_)
|
||||
| TxHistoryInfo::Pegout(_) => unreachable!(),
|
||||
};
|
||||
}
|
||||
|
||||
@ -326,54 +187,11 @@ impl Mempool {
|
||||
}
|
||||
|
||||
// Get all txids in the mempool
|
||||
pub fn txids(&self) -> Vec<&Txid> {
|
||||
pub fn txids(&self) -> Vec<&Sha256dHash> {
|
||||
let _timer = self.latency.with_label_values(&["txids"]).start_timer();
|
||||
self.txstore.keys().collect()
|
||||
}
|
||||
|
||||
// Get n txids after the given txid in the mempool
|
||||
pub fn txids_page(&self, n: usize, start: Option<Txid>) -> Vec<&Txid> {
|
||||
let _timer = self
|
||||
.latency
|
||||
.with_label_values(&["txids_page"])
|
||||
.start_timer();
|
||||
let start_bound = match start {
|
||||
Some(txid) => Excluded(txid),
|
||||
None => Unbounded,
|
||||
};
|
||||
|
||||
self.txstore
|
||||
.range((start_bound, Unbounded))
|
||||
.take(n)
|
||||
.map(|(k, _v)| k)
|
||||
.collect()
|
||||
}
|
||||
|
||||
// Get all txs in the mempool
|
||||
pub fn txs(&self) -> Vec<Transaction> {
|
||||
let _timer = self.latency.with_label_values(&["txs"]).start_timer();
|
||||
self.txstore.values().cloned().collect()
|
||||
}
|
||||
|
||||
// Get n txs after the given txid in the mempool
|
||||
pub fn txs_page(&self, n: usize, start: Option<Txid>) -> Vec<Transaction> {
|
||||
let _timer = self.latency.with_label_values(&["txs_page"]).start_timer();
|
||||
let mut page = Vec::with_capacity(n);
|
||||
let start_bound = match start {
|
||||
Some(txid) => Excluded(txid),
|
||||
None => Unbounded,
|
||||
};
|
||||
|
||||
self.txstore
|
||||
.range((start_bound, Unbounded))
|
||||
.take(n)
|
||||
.for_each(|(_, value)| {
|
||||
page.push(value.clone());
|
||||
});
|
||||
|
||||
page
|
||||
}
|
||||
|
||||
// Get an overview of the most recent transactions
|
||||
pub fn recent_txs_overview(&self) -> Vec<&TxOverview> {
|
||||
// We don't bother ever deleting elements from the recent list.
|
||||
@ -386,150 +204,100 @@ impl Mempool {
|
||||
&self.backlog_stats.0
|
||||
}
|
||||
|
||||
pub fn unique_txids(&self) -> HashSet<Txid> {
|
||||
HashSet::from_iter(self.txstore.keys().cloned())
|
||||
}
|
||||
|
||||
pub fn update(mempool: &RwLock<Mempool>, daemon: &Daemon) -> Result<()> {
|
||||
// 1. Start the metrics timer and get the current mempool txids
|
||||
// [LOCK] Takes read lock for whole scope.
|
||||
let (_timer, old_txids) = {
|
||||
let mempool = mempool.read().unwrap();
|
||||
(
|
||||
mempool.latency.with_label_values(&["update"]).start_timer(),
|
||||
mempool.unique_txids(),
|
||||
)
|
||||
};
|
||||
|
||||
// 2. Get all the mempool txids from the RPC.
|
||||
// [LOCK] No lock taken. Wait for RPC request. Get lists of remove/add txes.
|
||||
let all_txids = daemon
|
||||
pub fn update(&mut self, daemon: &Daemon) -> Result<()> {
|
||||
let _timer = self.latency.with_label_values(&["update"]).start_timer();
|
||||
let new_txids = daemon
|
||||
.getmempooltxids()
|
||||
.chain_err(|| "failed to update mempool from daemon")?;
|
||||
let txids_to_remove: HashSet<&Txid> = old_txids.difference(&all_txids).collect();
|
||||
let txids_to_add: Vec<&Txid> = all_txids.difference(&old_txids).collect();
|
||||
let old_txids = HashSet::from_iter(self.txstore.keys().cloned());
|
||||
let to_remove: HashSet<&Sha256dHash> = old_txids.difference(&new_txids).collect();
|
||||
|
||||
// 3. Remove missing transactions. Even if we are unable to download new transactions from
|
||||
// the daemon, we still want to remove the transactions that are no longer in the mempool.
|
||||
// [LOCK] Write lock is released at the end of the call to remove().
|
||||
mempool.write().unwrap().remove(txids_to_remove);
|
||||
|
||||
// 4. Download the new transactions from the daemon's mempool
|
||||
// [LOCK] No lock taken, waiting for RPC response.
|
||||
let txs_to_add = daemon
|
||||
.gettransactions(&txids_to_add)
|
||||
.chain_err(|| format!("failed to get {} transactions", txids_to_add.len()))?;
|
||||
|
||||
// 4. Update local mempool to match daemon's state
|
||||
// [LOCK] Takes Write lock for whole scope.
|
||||
{
|
||||
let mut mempool = mempool.write().unwrap();
|
||||
// Add new transactions
|
||||
if txs_to_add.len() > mempool.add(txs_to_add) {
|
||||
debug!("Mempool update added less transactions than expected");
|
||||
// Download and add new transactions from bitcoind's mempool
|
||||
let txids: Vec<&Sha256dHash> = new_txids.difference(&old_txids).collect();
|
||||
let to_add = match daemon.gettransactions(&txids) {
|
||||
Ok(txs) => txs,
|
||||
Err(err) => {
|
||||
warn!("failed to get transactions {:?}: {}", txids, err); // e.g. new block or RBF
|
||||
return Ok(()); // keep the mempool until next update()
|
||||
}
|
||||
};
|
||||
// Add new transactions
|
||||
self.add(to_add);
|
||||
// Remove missing transactions
|
||||
self.remove(to_remove);
|
||||
|
||||
mempool
|
||||
.count
|
||||
.with_label_values(&["txs"])
|
||||
.set(mempool.txstore.len() as f64);
|
||||
self.count
|
||||
.with_label_values(&["txs"])
|
||||
.set(self.txstore.len() as f64);
|
||||
|
||||
// Update cached backlog stats (if expired)
|
||||
if mempool.backlog_stats.1.elapsed()
|
||||
> Duration::from_secs(mempool.config.mempool_backlog_stats_ttl)
|
||||
{
|
||||
let _timer = mempool
|
||||
.latency
|
||||
.with_label_values(&["update_backlog_stats"])
|
||||
.start_timer();
|
||||
mempool.backlog_stats = (BacklogStats::new(&mempool.feeinfo), Instant::now());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
// Update cached backlog stats (if expired)
|
||||
if self.backlog_stats.1.elapsed() > Duration::from_secs(BACKLOG_STATS_TTL) {
|
||||
let _timer = self
|
||||
.latency
|
||||
.with_label_values(&["update_backlog_stats"])
|
||||
.start_timer();
|
||||
self.backlog_stats = (BacklogStats::new(&self.feeinfo), Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Txid) -> Result<()> {
|
||||
if !self.txstore.contains_key(txid) {
|
||||
if let Ok(tx) = daemon.getmempooltx(txid) {
|
||||
if self.add(vec![tx]) == 0 {
|
||||
return Err(format!(
|
||||
"Unable to add {txid} to mempool likely due to missing parents."
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Add transactions to the mempool.
|
||||
///
|
||||
/// The return value is the number of transactions processed.
|
||||
#[must_use = "Must deal with [[input vec's length]] > [[result]]."]
|
||||
fn add(&mut self, txs: Vec<Transaction>) -> usize {
|
||||
pub fn add_by_txid(&mut self, daemon: &Daemon, txid: &Sha256dHash) {
|
||||
if let Ok(tx) = daemon.getmempooltx(&txid) {
|
||||
self.add(vec![tx])
|
||||
}
|
||||
}
|
||||
|
||||
fn add(&mut self, txs: Vec<Transaction>) {
|
||||
self.delta
|
||||
.with_label_values(&["add"])
|
||||
.observe(txs.len() as f64);
|
||||
let _timer = self.latency.with_label_values(&["add"]).start_timer();
|
||||
let txlen = txs.len();
|
||||
if txlen == 0 {
|
||||
return 0;
|
||||
}
|
||||
debug!("Adding {} transactions to Mempool", txlen);
|
||||
|
||||
let mut txids = Vec::with_capacity(txs.len());
|
||||
let mut txids = vec![];
|
||||
// Phase 1: add to txstore
|
||||
for tx in txs {
|
||||
let txid = tx.get_txid();
|
||||
// Only push if it doesn't already exist.
|
||||
// This is important now that update doesn't lock during
|
||||
// the entire function body.
|
||||
if self.txstore.insert(txid, tx).is_none() {
|
||||
txids.push(txid);
|
||||
}
|
||||
let txid = tx.txid();
|
||||
txids.push(txid);
|
||||
self.txstore.insert(txid, tx);
|
||||
}
|
||||
|
||||
// Phase 2: index history and spend edges (some txos can be missing)
|
||||
let txos = self.lookup_txos(&self.get_prevouts(&txids));
|
||||
|
||||
// Count how many transactions were actually processed.
|
||||
let mut processed_count = 0;
|
||||
|
||||
// Phase 3: Iterate over the transactions and do the following:
|
||||
// 1. Find all of the TxOuts of each input parent using `txos`
|
||||
// 2. If any parent wasn't found, skip parsing this transaction
|
||||
// 3. Insert TxFeeInfo into info.
|
||||
// 4. Push TxOverview into recent tx queue.
|
||||
// 5. Create the Spend and Fund TxHistory structs for inputs + outputs
|
||||
// 6. Insert all TxHistory into history.
|
||||
// 7. Insert the tx edges into edges (HashMap of (Outpoint, (Txid, vin)))
|
||||
// 8. (Liquid only) Parse assets of tx.
|
||||
// Phase 2: index history and spend edges (can fail if some txos cannot be found)
|
||||
let txos = match self.lookup_txos(&self.get_prevouts(&txids)) {
|
||||
Ok(txos) => txos,
|
||||
Err(err) => {
|
||||
warn!("lookup txouts failed: {}", err);
|
||||
// TODO: should we remove txids from txstore?
|
||||
return;
|
||||
}
|
||||
};
|
||||
for txid in txids {
|
||||
let tx = self.txstore.get(&txid).expect("missing tx from txstore");
|
||||
|
||||
let prevouts = match extract_tx_prevouts(tx, &txos) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
warn!("Skipping tx {txid} missing parent error: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let tx = self.txstore.get(&txid).expect("missing mempool tx");
|
||||
let txid_bytes = full_hash(&txid[..]);
|
||||
|
||||
// Get feeinfo for caching and recent tx overview
|
||||
let feeinfo = TxFeeInfo::new(tx, &prevouts, self.config.network_type);
|
||||
let prevouts: HashMap<u32, &TxOut> = tx
|
||||
.input
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, txi)| has_prevout(txi))
|
||||
.map(|(index, txi)| {
|
||||
(
|
||||
index as u32,
|
||||
txos.get(&txi.previous_output)
|
||||
.expect(&format!("missing outpoint {:?}", txi.previous_output)),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// recent is an BoundedVecDeque that automatically evicts the oldest elements
|
||||
// Get feeinfo for caching and recent tx overview
|
||||
let feeinfo = TxFeeInfo::new(&tx, &prevouts);
|
||||
|
||||
// recent is an ArrayDeque that automatically evicts the oldest elements
|
||||
self.recent.push_front(TxOverview {
|
||||
txid,
|
||||
txid: txid,
|
||||
fee: feeinfo.fee,
|
||||
vsize: feeinfo.vsize,
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
value: prevouts
|
||||
.values()
|
||||
.map(|prevout| prevout.value.to_sat())
|
||||
.sum(),
|
||||
value: prevouts.values().map(|prevout| prevout.value).sum(),
|
||||
});
|
||||
|
||||
self.feeinfo.insert(txid, feeinfo);
|
||||
@ -537,82 +305,49 @@ impl Mempool {
|
||||
// An iterator over (ScriptHash, TxHistoryInfo)
|
||||
let spending = prevouts.into_iter().map(|(input_index, prevout)| {
|
||||
let txi = tx.input.get(input_index as usize).unwrap();
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
let value = prevout.value.to_sat();
|
||||
#[cfg(feature = "liquid")]
|
||||
let value = prevout.value;
|
||||
(
|
||||
compute_script_hash(&prevout.script_pubkey),
|
||||
TxHistoryInfo::Spending(SpendingInfo {
|
||||
txid: txid_bytes,
|
||||
vin: input_index,
|
||||
vin: input_index as u16,
|
||||
prev_txid: full_hash(&txi.previous_output.txid[..]),
|
||||
prev_vout: txi.previous_output.vout,
|
||||
value,
|
||||
prev_vout: txi.previous_output.vout as u16,
|
||||
value: prevout.value,
|
||||
}),
|
||||
)
|
||||
});
|
||||
|
||||
let config = &self.config;
|
||||
|
||||
// An iterator over (ScriptHash, TxHistoryInfo)
|
||||
let funding = tx
|
||||
.output
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, txo)| is_spendable(txo) || config.index_unspendables)
|
||||
.filter(|(_, txo)| is_spendable(txo))
|
||||
.map(|(index, txo)| {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
let value = txo.value.to_sat();
|
||||
#[cfg(feature = "liquid")]
|
||||
let value = txo.value;
|
||||
(
|
||||
compute_script_hash(&txo.script_pubkey),
|
||||
TxHistoryInfo::Funding(FundingInfo {
|
||||
txid: txid_bytes,
|
||||
vout: index as u32,
|
||||
value,
|
||||
vout: index as u16,
|
||||
value: txo.value,
|
||||
}),
|
||||
)
|
||||
});
|
||||
|
||||
// Index funding/spending history entries and spend edges
|
||||
for (scripthash, entry) in funding.chain(spending) {
|
||||
self.history.entry(scripthash).or_default().push(entry);
|
||||
self.history
|
||||
.entry(scripthash)
|
||||
.or_insert_with(|| Vec::new())
|
||||
.push(entry);
|
||||
}
|
||||
for (i, txi) in tx.input.iter().enumerate() {
|
||||
self.edges.insert(txi.previous_output, (txid, i as u32));
|
||||
}
|
||||
|
||||
// Index issued assets & native asset pegins/pegouts/burns
|
||||
#[cfg(feature = "liquid")]
|
||||
asset::index_mempool_tx_assets(
|
||||
tx,
|
||||
self.config.network_type,
|
||||
self.config.parent_network,
|
||||
&mut self.asset_history,
|
||||
&mut self.asset_issuance,
|
||||
);
|
||||
|
||||
processed_count += 1;
|
||||
}
|
||||
|
||||
processed_count
|
||||
}
|
||||
|
||||
/// Returns None if the lookup fails (mempool transaction RBF-ed etc.)
|
||||
pub fn lookup_txo(&self, outpoint: &OutPoint) -> Option<TxOut> {
|
||||
let mut outpoints = BTreeSet::new();
|
||||
outpoints.insert(*outpoint);
|
||||
// This can possibly be None now
|
||||
self.lookup_txos(&outpoints).remove(outpoint)
|
||||
}
|
||||
|
||||
/// For a given set of OutPoints, return a HashMap<OutPoint, TxOut>
|
||||
///
|
||||
/// Not all OutPoints from mempool transactions are guaranteed to be there.
|
||||
/// Ensure you deal with the None case in your logic.
|
||||
pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
|
||||
pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> Result<HashMap<OutPoint, TxOut>> {
|
||||
let _timer = self
|
||||
.latency
|
||||
.with_label_values(&["lookup_txos"])
|
||||
@ -623,29 +358,21 @@ impl Mempool {
|
||||
let mempool_txos = outpoints
|
||||
.iter()
|
||||
.filter(|outpoint| !confirmed_txos.contains_key(outpoint))
|
||||
.flat_map(|outpoint| {
|
||||
.map(|outpoint| {
|
||||
self.txstore
|
||||
.get(&outpoint.txid)
|
||||
.and_then(|tx| tx.output.get(outpoint.vout as usize).cloned())
|
||||
.map(|txout| (*outpoint, txout))
|
||||
.or_else(|| {
|
||||
warn!("missing outpoint {:?}", outpoint);
|
||||
None
|
||||
})
|
||||
.chain_err(|| format!("missing outpoint {:?}", outpoint))
|
||||
})
|
||||
.collect::<HashMap<OutPoint, TxOut>>();
|
||||
.collect::<Result<HashMap<OutPoint, TxOut>>>()?;
|
||||
|
||||
let mut txos = confirmed_txos;
|
||||
txos.extend(mempool_txos);
|
||||
txos
|
||||
Ok(txos)
|
||||
}
|
||||
|
||||
fn get_prevouts(&self, txids: &[Txid]) -> BTreeSet<OutPoint> {
|
||||
let _timer = self
|
||||
.latency
|
||||
.with_label_values(&["get_prevouts"])
|
||||
.start_timer();
|
||||
|
||||
fn get_prevouts(&self, txids: &[Sha256dHash]) -> BTreeSet<OutPoint> {
|
||||
txids
|
||||
.iter()
|
||||
.map(|txid| self.txstore.get(txid).expect("missing mempool tx"))
|
||||
@ -658,7 +385,7 @@ impl Mempool {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn remove(&mut self, to_remove: HashSet<&Txid>) {
|
||||
fn remove(&mut self, to_remove: HashSet<&Sha256dHash>) {
|
||||
self.delta
|
||||
.with_label_values(&["remove"])
|
||||
.observe(to_remove.len() as f64);
|
||||
@ -667,43 +394,20 @@ impl Mempool {
|
||||
for txid in &to_remove {
|
||||
self.txstore
|
||||
.remove(*txid)
|
||||
.unwrap_or_else(|| panic!("missing mempool tx {}", txid));
|
||||
|
||||
self.feeinfo.remove(*txid).or_else(|| {
|
||||
warn!("missing mempool tx feeinfo {}", txid);
|
||||
None
|
||||
});
|
||||
.expect(&format!("missing mempool tx {}", txid));
|
||||
self.feeinfo
|
||||
.remove(*txid)
|
||||
.expect(&format!("missing mempool tx feeinfo {}", txid));
|
||||
}
|
||||
|
||||
// TODO: make it more efficient (currently it takes O(|mempool|) time)
|
||||
self.history.retain(|_scripthash, entries| {
|
||||
entries.retain(|entry| !to_remove.contains(&entry.get_txid()));
|
||||
entries.retain(|entry| !to_remove.contains(&get_entry_txid(entry)));
|
||||
!entries.is_empty()
|
||||
});
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
asset::remove_mempool_tx_assets(
|
||||
&to_remove,
|
||||
&mut self.asset_history,
|
||||
&mut self.asset_issuance,
|
||||
);
|
||||
|
||||
self.edges
|
||||
.retain(|_outpoint, (txid, _vin)| !to_remove.contains(txid));
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn asset_history(&self, asset_id: &AssetId, limit: usize) -> Vec<Transaction> {
|
||||
let _timer = self
|
||||
.latency
|
||||
.with_label_values(&["asset_history"])
|
||||
.start_timer();
|
||||
self.asset_history
|
||||
.get(asset_id)
|
||||
.map_or_else(std::vec::Vec::new, |entries| {
|
||||
self._history(entries, None, limit)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@ -724,7 +428,7 @@ impl BacklogStats {
|
||||
}
|
||||
}
|
||||
|
||||
fn new(feeinfo: &HashMap<Txid, TxFeeInfo>) -> Self {
|
||||
fn new(feeinfo: &HashMap<Sha256dHash, TxFeeInfo>) -> Self {
|
||||
let (count, vsize, total_fee) = feeinfo
|
||||
.values()
|
||||
.fold((0, 0, 0), |(count, vsize, fee), feeinfo| {
|
||||
@ -739,3 +443,10 @@ impl BacklogStats {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_entry_txid(entry: &TxHistoryInfo) -> Sha256dHash {
|
||||
match entry {
|
||||
TxHistoryInfo::Funding(info) => parse_hash(&info.txid),
|
||||
TxHistoryInfo::Spending(info) => parse_hash(&info.txid),
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,25 +1,14 @@
|
||||
pub mod db;
|
||||
mod db;
|
||||
mod fetch;
|
||||
mod mempool;
|
||||
pub mod precache;
|
||||
mod query;
|
||||
pub mod schema;
|
||||
mod schema;
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub(crate) static THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
|
||||
rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(0) // 0 = use number of logical CPUs
|
||||
.thread_name(|i| format!("electrs-worker-{}", i))
|
||||
.build()
|
||||
.expect("failed to create global rayon thread pool")
|
||||
});
|
||||
|
||||
pub use self::db::{DBRow, DB};
|
||||
pub use self::fetch::{BlockEntry, FetchFrom};
|
||||
pub use self::mempool::Mempool;
|
||||
pub use self::query::Query;
|
||||
pub use self::schema::{
|
||||
compute_script_hash, parse_hash, ChainQuery, FundingInfo, Indexer, ScriptStats, SpendingInfo,
|
||||
SpendingInput, Store, TxHistoryInfo, TxHistoryKey, TxHistoryRow, Utxo,
|
||||
SpendingInput, Store, TxHistoryInfo, TxHistoryKey, Utxo,
|
||||
};
|
||||
|
||||
@ -1,59 +1,37 @@
|
||||
use crate::errors::*;
|
||||
use crate::new_index::ChainQuery;
|
||||
use crate::util::{full_hash, FullHash};
|
||||
use crate::util::{full_hash, Address, FullHash};
|
||||
|
||||
use crypto::digest::Digest;
|
||||
use crypto::sha2::Sha256;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use hex;
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::io::prelude::*;
|
||||
use std::sync::{atomic::AtomicUsize, Arc};
|
||||
use std::time::Instant;
|
||||
use std::str::FromStr;
|
||||
|
||||
pub fn precache(chain: Arc<ChainQuery>, scripthashes: Vec<FullHash>, threads: usize) {
|
||||
pub fn precache(chain: &ChainQuery, scripthashes: Vec<FullHash>) {
|
||||
let total = scripthashes.len();
|
||||
info!(
|
||||
"Pre-caching stats and utxo set on {} threads for {} scripthashes",
|
||||
threads, total
|
||||
);
|
||||
info!("Pre-caching stats and utxo set for {} scripthashes", total);
|
||||
|
||||
let pool = rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(threads)
|
||||
.num_threads(16)
|
||||
.thread_name(|i| format!("precache-{}", i))
|
||||
.build()
|
||||
.unwrap();
|
||||
let now = Instant::now();
|
||||
let counter = AtomicUsize::new(0);
|
||||
std::thread::spawn(move || {
|
||||
pool.install(|| {
|
||||
scripthashes
|
||||
.par_iter()
|
||||
.for_each(|scripthash| {
|
||||
// First, cache
|
||||
chain.stats(&scripthash[..], crate::new_index::db::DBFlush::Disable);
|
||||
let _ = chain.utxo(&scripthash[..], usize::MAX, crate::new_index::db::DBFlush::Disable);
|
||||
|
||||
// Then, increment the counter
|
||||
let pre_increment = counter.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
let post_increment_counter = pre_increment + 1;
|
||||
|
||||
// Then, log
|
||||
if post_increment_counter % 500 == 0 {
|
||||
let now_millis = now.elapsed().as_millis();
|
||||
info!("{post_increment_counter}/{total} Processed in {now_millis} ms running pre-cache for scripthash");
|
||||
}
|
||||
|
||||
// Every 10k counts, flush the DB to disk
|
||||
if post_increment_counter % 10000 == 0 {
|
||||
info!("Flushing cache_db... {post_increment_counter}");
|
||||
chain.store().cache_db().flush();
|
||||
info!("Done Flushing cache_db!!! {post_increment_counter}");
|
||||
}
|
||||
})
|
||||
});
|
||||
// After everything is done, flush the cache
|
||||
chain.store().cache_db().flush();
|
||||
pool.install(|| {
|
||||
scripthashes
|
||||
.par_iter()
|
||||
.enumerate()
|
||||
.for_each(|(i, scripthash)| {
|
||||
if i % 5 == 0 {
|
||||
debug!("Starting pre-cache for scripthash {}/{}", i + 1, total);
|
||||
}
|
||||
chain.stats(&scripthash[..]);
|
||||
//chain.utxo(&scripthash[..]);
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
@ -64,7 +42,34 @@ pub fn scripthashes_from_file(path: String) -> Result<Vec<FullHash>> {
|
||||
.lines()
|
||||
.map(|line| {
|
||||
let line = line.chain_err(|| "cannot read scripthash line")?;
|
||||
Ok(full_hash(&hex::decode(line).chain_err(|| "invalid hex")?))
|
||||
let cols: Vec<&str> = line.split(",").collect();
|
||||
to_scripthash(cols[0], cols[1])
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn to_scripthash(script_type: &str, script_str: &str) -> Result<FullHash> {
|
||||
match script_type {
|
||||
"address" => address_to_scripthash(script_str),
|
||||
"scripthash" => Ok(full_hash(
|
||||
&hex::decode(script_str).chain_err(|| "invalid hex")?,
|
||||
)),
|
||||
"scriptpubkey" => Ok(compute_script_hash(
|
||||
&hex::decode(script_str).chain_err(|| "invalid hex")?,
|
||||
)),
|
||||
_ => bail!("Invalid script type".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn address_to_scripthash(addr: &str) -> Result<FullHash> {
|
||||
let addr = Address::from_str(addr).chain_err(|| "invalid address")?;
|
||||
Ok(compute_script_hash(&addr.script_pubkey().as_bytes()))
|
||||
}
|
||||
|
||||
pub fn compute_script_hash(data: &[u8]) -> FullHash {
|
||||
let mut hash = FullHash::default();
|
||||
let mut sha2 = Sha256::new();
|
||||
sha2.input(data);
|
||||
sha2.result(&mut hash);
|
||||
hash
|
||||
}
|
||||
|
||||
@ -1,55 +1,31 @@
|
||||
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::sync::{Arc, RwLock, RwLockReadGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::chain::{Network, OutPoint, Transaction, TxOut, Txid, TxidCompat};
|
||||
use crate::config::Config;
|
||||
use crate::daemon::{Daemon, MempoolAcceptResult, SubmitPackageResult};
|
||||
use crate::chain::{OutPoint, Transaction, TxOut};
|
||||
use crate::daemon::Daemon;
|
||||
use crate::errors::*;
|
||||
use crate::new_index::{ChainQuery, Mempool, ScriptStats, SpendingInput, Utxo};
|
||||
use crate::util::{is_spendable, BlockId, Bytes, TransactionStatus};
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
use crate::{
|
||||
chain::{asset::AssetRegistryLock, AssetId},
|
||||
elements::{lookup_asset, AssetMeta, AssetRegistry, AssetSorting, LiquidAsset},
|
||||
};
|
||||
|
||||
const FEE_ESTIMATES_TTL: u64 = 60; // seconds
|
||||
|
||||
const CONF_TARGETS: [u16; 28] = [
|
||||
1u16, 2u16, 3u16, 4u16, 5u16, 6u16, 7u16, 8u16, 9u16, 10u16, 11u16, 12u16, 13u16, 14u16, 15u16,
|
||||
16u16, 17u16, 18u16, 19u16, 20u16, 21u16, 22u16, 23u16, 24u16, 25u16, 144u16, 504u16, 1008u16,
|
||||
const CONF_TARGETS: [u16; 9] = [
|
||||
2u16, 3u16, 4u16, 6u16, 10u16, 20u16, 144u16, 504u16, 1008u16,
|
||||
];
|
||||
|
||||
pub struct Query {
|
||||
chain: Arc<ChainQuery>, // TODO: should be used as read-only
|
||||
mempool: Arc<RwLock<Mempool>>,
|
||||
daemon: Arc<Daemon>,
|
||||
config: Arc<Config>,
|
||||
cached_estimates: RwLock<(HashMap<u16, f64>, Option<Instant>)>,
|
||||
cached_relayfee: RwLock<Option<f64>>,
|
||||
#[cfg(feature = "liquid")]
|
||||
asset_db: Option<Arc<RwLock<AssetRegistry>>>,
|
||||
}
|
||||
|
||||
impl Query {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
pub fn new(
|
||||
chain: Arc<ChainQuery>,
|
||||
mempool: Arc<RwLock<Mempool>>,
|
||||
daemon: Arc<Daemon>,
|
||||
config: Arc<Config>,
|
||||
) -> Self {
|
||||
pub fn new(chain: Arc<ChainQuery>, mempool: Arc<RwLock<Mempool>>, daemon: Arc<Daemon>) -> Self {
|
||||
Query {
|
||||
chain,
|
||||
mempool,
|
||||
daemon,
|
||||
config,
|
||||
cached_estimates: RwLock::new((HashMap::new(), None)),
|
||||
cached_relayfee: RwLock::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
@ -57,73 +33,37 @@ impl Query {
|
||||
&self.chain
|
||||
}
|
||||
|
||||
pub fn config(&self) -> &Config {
|
||||
&self.config
|
||||
}
|
||||
|
||||
pub fn network(&self) -> Network {
|
||||
self.config.network_type
|
||||
}
|
||||
|
||||
pub fn mempool(&self) -> RwLockReadGuard<'_, Mempool> {
|
||||
pub fn mempool(&self) -> RwLockReadGuard<Mempool> {
|
||||
self.mempool.read().unwrap()
|
||||
}
|
||||
|
||||
pub fn broadcast_raw(&self, txhex: &str) -> Result<Txid> {
|
||||
let txid = self.daemon.broadcast_raw(txhex)?;
|
||||
// The important part is whether we succeeded in broadcasting.
|
||||
// Ignore errors in adding to the cache and show an internal warning.
|
||||
if let Err(e) = self
|
||||
.mempool
|
||||
pub fn broadcast_raw(&self, txhex: &String) -> Result<Sha256dHash> {
|
||||
let txid = self.daemon.broadcast_raw(&txhex)?;
|
||||
self.mempool
|
||||
.write()
|
||||
.unwrap()
|
||||
.add_by_txid(&self.daemon, &txid)
|
||||
{
|
||||
warn!(
|
||||
"broadcast_raw of {txid} succeeded to broadcast \
|
||||
but failed to add to mempool-electrs Mempool cache: {e}"
|
||||
);
|
||||
}
|
||||
.add_by_txid(&self.daemon, &txid);
|
||||
Ok(txid)
|
||||
}
|
||||
|
||||
pub fn test_mempool_accept(
|
||||
&self,
|
||||
txhex: Vec<String>,
|
||||
maxfeerate: Option<f64>,
|
||||
) -> Result<Vec<MempoolAcceptResult>> {
|
||||
self.daemon.test_mempool_accept(txhex, maxfeerate)
|
||||
}
|
||||
|
||||
pub fn submit_package(
|
||||
&self,
|
||||
txhex: Vec<String>,
|
||||
maxfeerate: Option<f64>,
|
||||
maxburnamount: Option<f64>,
|
||||
) -> Result<SubmitPackageResult> {
|
||||
self.daemon.submit_package(txhex, maxfeerate, maxburnamount)
|
||||
}
|
||||
|
||||
pub fn utxo(&self, scripthash: &[u8]) -> Result<Vec<Utxo>> {
|
||||
let mut utxos = self.chain.utxo(
|
||||
scripthash,
|
||||
self.config.utxos_limit,
|
||||
super::db::DBFlush::Enable,
|
||||
)?;
|
||||
pub fn utxo(&self, scripthash: &[u8]) -> Vec<Utxo> {
|
||||
let mut utxos = self.chain.utxo(scripthash);
|
||||
let mempool = self.mempool();
|
||||
utxos.retain(|utxo| !mempool.has_spend(&OutPoint::from(utxo)));
|
||||
utxos.extend(mempool.utxo(scripthash));
|
||||
Ok(utxos)
|
||||
utxos
|
||||
}
|
||||
|
||||
pub fn history_txids(&self, scripthash: &[u8], limit: usize) -> Vec<(Txid, Option<BlockId>)> {
|
||||
let confirmed_txids = self.chain.history_txids(scripthash, limit);
|
||||
let confirmed_len = confirmed_txids.len();
|
||||
let confirmed_txids = confirmed_txids.into_iter().map(|(tx, b)| (tx, Some(b)));
|
||||
pub fn history_txids(&self, scripthash: &[u8]) -> Vec<(Sha256dHash, Option<BlockId>)> {
|
||||
let confirmed_txids = self
|
||||
.chain
|
||||
.history_txids(scripthash)
|
||||
.into_iter()
|
||||
.map(|(tx, b)| (tx, Some(b)));
|
||||
|
||||
let mempool_txids = self
|
||||
.mempool()
|
||||
.history_txids(scripthash, limit - confirmed_len)
|
||||
.history_txids(scripthash)
|
||||
.into_iter()
|
||||
.map(|tx| (tx, None));
|
||||
|
||||
@ -132,26 +72,27 @@ impl Query {
|
||||
|
||||
pub fn stats(&self, scripthash: &[u8]) -> (ScriptStats, ScriptStats) {
|
||||
(
|
||||
self.chain.stats(scripthash, super::db::DBFlush::Enable),
|
||||
self.chain.stats(scripthash),
|
||||
self.mempool().stats(scripthash),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn lookup_txn(&self, txid: &Txid) -> Option<Transaction> {
|
||||
pub fn lookup_txn(&self, txid: &Sha256dHash) -> Option<Transaction> {
|
||||
self.chain
|
||||
.lookup_txn(txid, None)
|
||||
.lookup_txn(txid)
|
||||
.or_else(|| self.mempool().lookup_txn(txid))
|
||||
}
|
||||
pub fn lookup_raw_txn(&self, txid: &Txid) -> Option<Bytes> {
|
||||
pub fn lookup_raw_txn(&self, txid: &Sha256dHash) -> Option<Bytes> {
|
||||
self.chain
|
||||
.lookup_raw_txn(txid, None)
|
||||
.lookup_raw_txn(txid)
|
||||
.or_else(|| self.mempool().lookup_raw_txn(txid))
|
||||
}
|
||||
|
||||
/// Not all OutPoints from mempool transactions are guaranteed to be included in the result
|
||||
pub fn lookup_txos(&self, outpoints: &BTreeSet<OutPoint>) -> HashMap<OutPoint, TxOut> {
|
||||
// the mempool lookup_txos() internally looks up confirmed txos as well
|
||||
self.mempool().lookup_txos(outpoints)
|
||||
self.mempool()
|
||||
.lookup_txos(outpoints)
|
||||
.expect("failed loading txos")
|
||||
}
|
||||
|
||||
pub fn lookup_spend(&self, outpoint: &OutPoint) -> Option<SpendingInput> {
|
||||
@ -161,7 +102,7 @@ impl Query {
|
||||
}
|
||||
|
||||
pub fn lookup_tx_spends(&self, tx: Transaction) -> Vec<Option<SpendingInput>> {
|
||||
let txid = tx.get_txid();
|
||||
let txid = tx.txid();
|
||||
|
||||
tx.output
|
||||
.par_iter()
|
||||
@ -179,155 +120,23 @@ impl Query {
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn get_tx_status(&self, txid: &Txid) -> TransactionStatus {
|
||||
pub fn get_tx_status(&self, txid: &Sha256dHash) -> TransactionStatus {
|
||||
TransactionStatus::from(self.chain.tx_confirming_block(txid))
|
||||
}
|
||||
|
||||
pub fn get_mempool_tx_fee(&self, txid: &Txid) -> Option<u64> {
|
||||
self.mempool().get_tx_fee(txid)
|
||||
// TODO cache, only allow getting estimatess for cached items
|
||||
pub fn estimate_fee(&self, conf_target: u16) -> Option<f32> {
|
||||
self.daemon.estimatesmartfee(conf_target).ok()
|
||||
}
|
||||
|
||||
pub fn has_unconfirmed_parents(&self, txid: &Txid) -> bool {
|
||||
self.mempool().has_unconfirmed_parents(txid)
|
||||
}
|
||||
|
||||
pub fn estimate_fee(&self, conf_target: u16) -> Option<f64> {
|
||||
if self.config.network_type.is_regtest() {
|
||||
return self.get_relayfee().ok();
|
||||
}
|
||||
if let (ref cache, Some(cache_time)) = *self.cached_estimates.read().unwrap() {
|
||||
if cache_time.elapsed() < Duration::from_secs(FEE_ESTIMATES_TTL) {
|
||||
return cache.get(&conf_target).copied();
|
||||
}
|
||||
}
|
||||
|
||||
self.update_fee_estimates();
|
||||
self.cached_estimates
|
||||
.read()
|
||||
.unwrap()
|
||||
.0
|
||||
.get(&conf_target)
|
||||
.copied()
|
||||
}
|
||||
|
||||
pub fn estimate_fee_map(&self) -> HashMap<u16, f64> {
|
||||
if let (ref cache, Some(cache_time)) = *self.cached_estimates.read().unwrap() {
|
||||
if cache_time.elapsed() < Duration::from_secs(FEE_ESTIMATES_TTL) {
|
||||
return cache.clone();
|
||||
}
|
||||
}
|
||||
|
||||
self.update_fee_estimates();
|
||||
self.cached_estimates.read().unwrap().0.clone()
|
||||
}
|
||||
|
||||
fn update_fee_estimates(&self) {
|
||||
match self.daemon.estimatesmartfee_batch(&CONF_TARGETS) {
|
||||
Ok(estimates) => {
|
||||
*self.cached_estimates.write().unwrap() = (estimates, Some(Instant::now()));
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("failed estimating feerates: {:?}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_relayfee(&self) -> Result<f64> {
|
||||
if let Some(cached) = *self.cached_relayfee.read().unwrap() {
|
||||
return Ok(cached);
|
||||
}
|
||||
|
||||
let relayfee = self.daemon.get_relayfee()?;
|
||||
self.cached_relayfee.write().unwrap().replace(relayfee);
|
||||
Ok(relayfee)
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn new(
|
||||
chain: Arc<ChainQuery>,
|
||||
mempool: Arc<RwLock<Mempool>>,
|
||||
daemon: Arc<Daemon>,
|
||||
config: Arc<Config>,
|
||||
asset_db: Option<Arc<RwLock<AssetRegistry>>>,
|
||||
) -> Self {
|
||||
Query {
|
||||
chain,
|
||||
mempool,
|
||||
daemon,
|
||||
config,
|
||||
asset_db,
|
||||
cached_estimates: RwLock::new((HashMap::new(), None)),
|
||||
cached_relayfee: RwLock::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn lookup_asset(&self, asset_id: &AssetId) -> Result<Option<LiquidAsset>> {
|
||||
lookup_asset(
|
||||
self,
|
||||
self.asset_db.as_ref().map(AssetRegistryLock::RwLock),
|
||||
asset_id,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn lookup_registry_asset(&self, asset_id: &AssetId) -> Result<Option<AssetMeta>> {
|
||||
let asset_db = self
|
||||
.asset_db
|
||||
.as_ref()
|
||||
.chain_err(|| "asset registry unavailable")?;
|
||||
Ok(asset_db.read().unwrap().get(asset_id).cloned())
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn list_registry_assets(
|
||||
&self,
|
||||
start_index: usize,
|
||||
limit: usize,
|
||||
sorting: AssetSorting,
|
||||
) -> Result<(usize, Vec<LiquidAsset>)> {
|
||||
let asset_db = match &self.asset_db {
|
||||
None => return Ok((0, vec![])),
|
||||
Some(db) => db.read().unwrap(),
|
||||
};
|
||||
let (total_num, results) = asset_db.list(start_index, limit, sorting);
|
||||
// Attach on-chain information alongside the registry metadata
|
||||
let results = results
|
||||
.into_iter()
|
||||
.map(|(asset_id, metadata)| {
|
||||
lookup_asset(
|
||||
self,
|
||||
Some(AssetRegistryLock::RwLockReadGuard(&asset_db)),
|
||||
asset_id,
|
||||
Some(metadata),
|
||||
)?
|
||||
.chain_err(|| "missing registered asset")
|
||||
// TODO cache
|
||||
pub fn estimate_fee_targets(&self) -> HashMap<u16, f32> {
|
||||
CONF_TARGETS
|
||||
.iter()
|
||||
.filter_map(|conf_target| {
|
||||
self.estimate_fee(*conf_target)
|
||||
.map(|feerate| (*conf_target, feerate))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
Ok((total_num, results))
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn search_registry_assets<T, F>(
|
||||
&self,
|
||||
search: &str,
|
||||
limit: usize,
|
||||
mut map: F,
|
||||
) -> Result<Vec<T>>
|
||||
where
|
||||
F: FnMut(&AssetId, &AssetMeta) -> T,
|
||||
{
|
||||
let asset_db = self
|
||||
.asset_db
|
||||
.as_ref()
|
||||
.chain_err(|| "asset registry unavailable")?;
|
||||
Ok(asset_db
|
||||
.read()
|
||||
.unwrap()
|
||||
.search(search, limit)
|
||||
.into_iter()
|
||||
.map(|(asset_id, metadata)| map(asset_id, metadata))
|
||||
.collect())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
1956
src/rest.rs
1956
src/rest.rs
File diff suppressed because it is too large
Load Diff
@ -1,58 +1,34 @@
|
||||
use crossbeam_channel as channel;
|
||||
use crossbeam_channel::RecvTimeoutError;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use signal_hook::consts::{SIGINT, SIGTERM, SIGUSR1};
|
||||
use chan;
|
||||
use chan_signal;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::errors::*;
|
||||
|
||||
#[derive(Clone)] // so multiple threads could wait on signals
|
||||
pub struct Waiter {
|
||||
receiver: channel::Receiver<i32>,
|
||||
}
|
||||
|
||||
fn notify(signals: &[i32]) -> channel::Receiver<i32> {
|
||||
let (s, r) = channel::bounded(1);
|
||||
let mut signals =
|
||||
signal_hook::iterator::Signals::new(signals).expect("failed to register signal hook");
|
||||
crate::util::spawn_thread("signal-notifier", move || {
|
||||
for signal in signals.forever() {
|
||||
s.send(signal)
|
||||
.unwrap_or_else(|_| panic!("failed to send signal {}", signal));
|
||||
}
|
||||
});
|
||||
r
|
||||
signal: chan::Receiver<chan_signal::Signal>,
|
||||
}
|
||||
|
||||
impl Waiter {
|
||||
pub fn start() -> Waiter {
|
||||
pub fn new() -> Waiter {
|
||||
Waiter {
|
||||
receiver: notify(&[
|
||||
SIGINT, SIGTERM,
|
||||
SIGUSR1, // allow external triggering (e.g. via bitcoind `blocknotify`)
|
||||
]),
|
||||
signal: chan_signal::notify(&[chan_signal::Signal::INT, chan_signal::Signal::TERM]),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn wait(&self, duration: Duration, accept_sigusr: bool) -> Result<()> {
|
||||
// Determine the deadline time based on the duration, so that it doesn't
|
||||
// get pushed back when wait_deadline() recurses
|
||||
self.wait_deadline(Instant::now() + duration, accept_sigusr)
|
||||
}
|
||||
|
||||
fn wait_deadline(&self, deadline: Instant, accept_sigusr: bool) -> Result<()> {
|
||||
match self.receiver.recv_deadline(deadline) {
|
||||
Ok(sig) if sig == SIGUSR1 => {
|
||||
trace!("notified via SIGUSR1");
|
||||
if accept_sigusr {
|
||||
Ok(())
|
||||
} else {
|
||||
self.wait_deadline(deadline, accept_sigusr)
|
||||
pub fn wait(&self, duration: Duration) -> Result<()> {
|
||||
let signal = &self.signal;
|
||||
let timeout = chan::after(duration);
|
||||
chan_select! {
|
||||
signal.recv() -> s => {
|
||||
if let Some(sig) = s {
|
||||
bail!(ErrorKind::Interrupt(sig));
|
||||
}
|
||||
}
|
||||
Ok(sig) => bail!(ErrorKind::Interrupt(sig)),
|
||||
Err(RecvTimeoutError::Timeout) => Ok(()),
|
||||
Err(RecvTimeoutError::Disconnected) => bail!("signal hook channel disconnected"),
|
||||
},
|
||||
timeout.recv() => {},
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn poll(&self) -> Result<()> {
|
||||
self.wait(Duration::from_secs(0))
|
||||
}
|
||||
}
|
||||
|
||||
320
src/util/address.rs
Normal file
320
src/util/address.rs
Normal file
@ -0,0 +1,320 @@
|
||||
// Rust Bitcoin Library
|
||||
// Written in 2014 by
|
||||
// Andrew Poelstra <apoelstra@wpsoftware.net>
|
||||
// To the extent possible under law, the author(s) have dedicated all
|
||||
// copyright and related and neighboring rights to this software to
|
||||
// the public domain worldwide. This software is distributed without
|
||||
// any warranty.
|
||||
//
|
||||
// You should have received a copy of the CC0 Public Domain Dedication
|
||||
// along with this software.
|
||||
// If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
|
||||
//
|
||||
|
||||
//! Addresses
|
||||
//!
|
||||
//! Support for ordinary base58 Bitcoin addresses and private keys
|
||||
//!
|
||||
|
||||
use std::fmt::{self, Display, Formatter};
|
||||
use std::str::FromStr;
|
||||
|
||||
use bitcoin_bech32::{self, u5, WitnessProgram};
|
||||
use bitcoin_hashes::{hash160, Hash};
|
||||
|
||||
use crate::chain::Network;
|
||||
use bitcoin::blockdata::opcodes;
|
||||
use bitcoin::blockdata::script;
|
||||
use bitcoin::consensus::encode;
|
||||
use bitcoin::util::base58;
|
||||
use bitcoin::util::key;
|
||||
|
||||
/// The method used to produce an address
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum Payload {
|
||||
/// pay-to-pkhash address
|
||||
PubkeyHash(hash160::Hash),
|
||||
/// P2SH address
|
||||
ScriptHash(hash160::Hash),
|
||||
/// Segwit address
|
||||
WitnessProgram(WitnessProgram),
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
/// A Bitcoin address
|
||||
pub struct Address {
|
||||
/// The type of the address
|
||||
pub payload: Payload,
|
||||
/// The network on which this address is usable
|
||||
pub network: Network,
|
||||
}
|
||||
|
||||
impl Address {
|
||||
/// Creates a pay to (compressed) public key hash address from a public key
|
||||
/// This is the preferred non-witness type address
|
||||
#[inline]
|
||||
pub fn p2pkh(pk: &key::PublicKey, network: Network) -> Address {
|
||||
let mut hash_engine = hash160::Hash::engine();
|
||||
pk.write_into(&mut hash_engine);
|
||||
|
||||
Address {
|
||||
network: network,
|
||||
payload: Payload::PubkeyHash(hash160::Hash::from_engine(hash_engine)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a pay to script hash P2SH address from a script
|
||||
/// This address type was introduced with BIP16 and is the popular type to implement multi-sig these days.
|
||||
#[inline]
|
||||
pub fn p2sh(script: &script::Script, network: Network) -> Address {
|
||||
Address {
|
||||
network: network,
|
||||
payload: Payload::ScriptHash(hash160::Hash::hash(&script[..])),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a witness pay to public key address from a public key
|
||||
/// This is the native segwit address type for an output redeemable with a single signature
|
||||
pub fn p2wpkh(pk: &key::PublicKey, network: Network) -> Address {
|
||||
let mut hash_engine = hash160::Hash::engine();
|
||||
pk.write_into(&mut hash_engine);
|
||||
|
||||
Address {
|
||||
network: network,
|
||||
payload: Payload::WitnessProgram(
|
||||
// unwrap is safe as witness program is known to be correct as above
|
||||
WitnessProgram::new(
|
||||
u5::try_from_u8(0).expect("0<32"),
|
||||
hash160::Hash::from_engine(hash_engine)[..].to_vec(),
|
||||
Address::bech_network(network),
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a pay to script address that embeds a witness pay to public key
|
||||
/// This is a segwit address type that looks familiar (as p2sh) to legacy clients
|
||||
pub fn p2shwpkh(pk: &key::PublicKey, network: Network) -> Address {
|
||||
let mut hash_engine = hash160::Hash::engine();
|
||||
pk.write_into(&mut hash_engine);
|
||||
|
||||
let builder = script::Builder::new()
|
||||
.push_int(0)
|
||||
.push_slice(&hash160::Hash::from_engine(hash_engine)[..]);
|
||||
|
||||
Address {
|
||||
network: network,
|
||||
payload: Payload::ScriptHash(hash160::Hash::hash(builder.into_script().as_bytes())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a witness pay to script hash address
|
||||
pub fn p2wsh(script: &script::Script, network: Network) -> Address {
|
||||
use bitcoin_hashes::sha256;
|
||||
use bitcoin_hashes::Hash;
|
||||
|
||||
Address {
|
||||
network: network,
|
||||
payload: Payload::WitnessProgram(
|
||||
// unwrap is safe as witness program is known to be correct as above
|
||||
WitnessProgram::new(
|
||||
u5::try_from_u8(0).expect("0<32"),
|
||||
sha256::Hash::hash(&script[..])[..].to_vec(),
|
||||
Address::bech_network(network),
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a pay to script address that embeds a witness pay to script hash address
|
||||
/// This is a segwit address type that looks familiar (as p2sh) to legacy clients
|
||||
pub fn p2shwsh(script: &script::Script, network: Network) -> Address {
|
||||
use bitcoin_hashes::hash160;
|
||||
use bitcoin_hashes::sha256;
|
||||
use bitcoin_hashes::Hash;
|
||||
|
||||
let ws = script::Builder::new()
|
||||
.push_int(0)
|
||||
.push_slice(&sha256::Hash::hash(&script[..])[..])
|
||||
.into_script();
|
||||
|
||||
Address {
|
||||
network: network,
|
||||
payload: Payload::ScriptHash(hash160::Hash::hash(&ws[..])),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
/// convert Network to bech32 network (this should go away soon)
|
||||
fn bech_network(network: Network) -> bitcoin_bech32::constants::Network {
|
||||
match network {
|
||||
Network::Bitcoin => bitcoin_bech32::constants::Network::Bitcoin,
|
||||
Network::Testnet => bitcoin_bech32::constants::Network::Testnet,
|
||||
Network::Regtest => bitcoin_bech32::constants::Network::Regtest,
|
||||
|
||||
// this should never actually happen, Liquid does not have bech32 addresses
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::Liquid | Network::LiquidRegtest => bitcoin_bech32::constants::Network::Bitcoin,
|
||||
}
|
||||
}
|
||||
|
||||
/// Generates a script pubkey spending to this address
|
||||
pub fn script_pubkey(&self) -> script::Script {
|
||||
match self.payload {
|
||||
Payload::PubkeyHash(ref hash) => script::Builder::new()
|
||||
.push_opcode(opcodes::all::OP_DUP)
|
||||
.push_opcode(opcodes::all::OP_HASH160)
|
||||
.push_slice(&hash[..])
|
||||
.push_opcode(opcodes::all::OP_EQUALVERIFY)
|
||||
.push_opcode(opcodes::all::OP_CHECKSIG),
|
||||
Payload::ScriptHash(ref hash) => script::Builder::new()
|
||||
.push_opcode(opcodes::all::OP_HASH160)
|
||||
.push_slice(&hash[..])
|
||||
.push_opcode(opcodes::all::OP_EQUAL),
|
||||
Payload::WitnessProgram(ref witprog) => script::Builder::new()
|
||||
.push_int(witprog.version().to_u8() as i64)
|
||||
.push_slice(witprog.program()),
|
||||
}
|
||||
.into_script()
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for Address {
|
||||
fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
|
||||
match self.payload {
|
||||
Payload::PubkeyHash(ref hash) => {
|
||||
let mut prefixed = [0; 21];
|
||||
prefixed[0] = match self.network {
|
||||
Network::Bitcoin => 0,
|
||||
Network::Testnet | Network::Regtest => 111,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::Liquid => 57,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidRegtest => 235,
|
||||
};
|
||||
prefixed[1..].copy_from_slice(&hash[..]);
|
||||
base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
|
||||
}
|
||||
Payload::ScriptHash(ref hash) => {
|
||||
let mut prefixed = [0; 21];
|
||||
prefixed[0] = match self.network {
|
||||
Network::Bitcoin => 5,
|
||||
Network::Testnet | Network::Regtest => 196,
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::Liquid => 39,
|
||||
#[cfg(feature = "liquid")]
|
||||
Network::LiquidRegtest => 75,
|
||||
};
|
||||
prefixed[1..].copy_from_slice(&hash[..]);
|
||||
base58::check_encode_slice_to_fmt(fmt, &prefixed[..])
|
||||
}
|
||||
Payload::WitnessProgram(ref witprog) => fmt.write_str(&witprog.to_address()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for Address {
|
||||
type Err = encode::Error;
|
||||
|
||||
fn from_str(s: &str) -> Result<Address, encode::Error> {
|
||||
// bech32 (note that upper or lowercase is allowed but NOT mixed case)
|
||||
if s.starts_with("bc1")
|
||||
|| s.starts_with("BC1")
|
||||
|| s.starts_with("tb1")
|
||||
|| s.starts_with("TB1")
|
||||
|| s.starts_with("bcrt1")
|
||||
|| s.starts_with("BCRT1")
|
||||
{
|
||||
let witprog = WitnessProgram::from_address(s)?;
|
||||
let network = match witprog.network() {
|
||||
bitcoin_bech32::constants::Network::Bitcoin => Network::Bitcoin,
|
||||
bitcoin_bech32::constants::Network::Testnet => Network::Testnet,
|
||||
bitcoin_bech32::constants::Network::Regtest => Network::Regtest,
|
||||
_ => panic!("unknown network"),
|
||||
};
|
||||
if witprog.version().to_u8() != 0 {
|
||||
return Err(encode::Error::UnsupportedWitnessVersion(
|
||||
witprog.version().to_u8(),
|
||||
));
|
||||
}
|
||||
return Ok(Address {
|
||||
network: network,
|
||||
payload: Payload::WitnessProgram(witprog),
|
||||
});
|
||||
}
|
||||
|
||||
if s.len() > 50 {
|
||||
return Err(encode::Error::Base58(base58::Error::InvalidLength(
|
||||
s.len() * 11 / 15,
|
||||
)));
|
||||
}
|
||||
|
||||
// Base 58
|
||||
let data = base58::from_check(s)?;
|
||||
|
||||
if data.len() != 21 {
|
||||
return Err(encode::Error::Base58(base58::Error::InvalidLength(
|
||||
data.len(),
|
||||
)));
|
||||
}
|
||||
|
||||
let (network, payload) = match data[0] {
|
||||
0 => (
|
||||
Network::Bitcoin,
|
||||
Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
|
||||
),
|
||||
5 => (
|
||||
Network::Bitcoin,
|
||||
Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
|
||||
),
|
||||
111 => (
|
||||
Network::Testnet,
|
||||
Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
|
||||
),
|
||||
196 => (
|
||||
Network::Testnet,
|
||||
Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
|
||||
),
|
||||
#[cfg(feature = "liquid")]
|
||||
57 => (
|
||||
Network::Liquid,
|
||||
Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
|
||||
),
|
||||
#[cfg(feature = "liquid")]
|
||||
39 => (
|
||||
Network::Liquid,
|
||||
Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
|
||||
),
|
||||
#[cfg(feature = "liquid")]
|
||||
235 => (
|
||||
Network::LiquidRegtest,
|
||||
Payload::PubkeyHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
|
||||
),
|
||||
#[cfg(feature = "liquid")]
|
||||
75 => (
|
||||
Network::LiquidRegtest,
|
||||
Payload::ScriptHash(hash160::Hash::from_slice(&data[1..]).unwrap()),
|
||||
),
|
||||
x => {
|
||||
return Err(encode::Error::Base58(base58::Error::InvalidVersion(vec![
|
||||
x,
|
||||
])));
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Address {
|
||||
network: network,
|
||||
payload: payload,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ::std::fmt::Debug for Address {
|
||||
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
|
||||
write!(f, "{}", self.to_string())
|
||||
}
|
||||
}
|
||||
@ -1,112 +0,0 @@
|
||||
/*
|
||||
|
||||
The tests below show us the following defaults for each method of using bincode.
|
||||
|
||||
1. Using bincode::[de]serialize() directly: "function"
|
||||
2. Using bincode::config().[de]serialize(): "Config" (deprecated)
|
||||
3. Using bincode::options().[de]serialize(): "Options" (currently recommended for v1.3.3)
|
||||
|
||||
```
|
||||
+----------+--------+------------+----------------+------------+
|
||||
| | Endian | Int Length | Allow Trailing | Byte Limit |
|
||||
+----------+--------+------------+----------------+------------+
|
||||
| function | little | fixed | allow | unlimited |
|
||||
| Config | little | fixed | allow | unlimited |
|
||||
| Options | little | variable * | reject * | unlimited |
|
||||
+----------+--------+------------+----------------+------------+
|
||||
```
|
||||
|
||||
Thus we only need to change the int length from variable to fixed,
|
||||
and allow trailing to allow in order to match the previous behavior.
|
||||
(note: TxHistory was using Big Endian by explicitly setting it to big.)
|
||||
|
||||
*/
|
||||
|
||||
use bincode_do_not_use_directly as bincode;
|
||||
|
||||
#[test]
|
||||
fn bincode_settings() {
|
||||
let value = TestStruct::new();
|
||||
let mut large = [0_u8; 4096];
|
||||
let decoded = [
|
||||
8_u8, 7, 6, 5, 4, 3, 2, 1, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 8, 7, 6, 5, 4, 3, 2, 1, 1,
|
||||
2, 3, 4, 5, 6, 7, 8, 12, 0, 0, 0, 0, 0, 0, 0, 72, 101, 108, 108, 111, 32, 87, 111, 114,
|
||||
108, 100, 33,
|
||||
];
|
||||
large[0..56].copy_from_slice(&decoded);
|
||||
|
||||
// Using functions: Little endian, Fixint, Allow trailing, Unlimited
|
||||
assert_eq!(bincode::serialize(&value).unwrap(), &decoded);
|
||||
assert_eq!(bincode::deserialize::<TestStruct>(&large).unwrap(), value);
|
||||
|
||||
// Using Config (deprecated)
|
||||
// Little endian, fixint, Allow trailing, Unlimited
|
||||
#[allow(deprecated)]
|
||||
{
|
||||
assert_eq!(bincode::config().serialize(&value).unwrap(), &decoded);
|
||||
assert_eq!(
|
||||
bincode::config().deserialize::<TestStruct>(&large).unwrap(),
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
// Using Options
|
||||
// Little endian, VARINT (different), Reject trailing (different), unlimited
|
||||
use bincode::Options;
|
||||
assert_eq!(
|
||||
bincode::options()
|
||||
.with_fixint_encoding()
|
||||
.allow_trailing_bytes()
|
||||
.serialize(&value)
|
||||
.unwrap(),
|
||||
&decoded
|
||||
);
|
||||
assert_eq!(
|
||||
bincode::options()
|
||||
.with_fixint_encoding()
|
||||
.allow_trailing_bytes()
|
||||
.deserialize::<TestStruct>(&large)
|
||||
.unwrap(),
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
struct TestStruct {
|
||||
a: u64,
|
||||
b: [u8; 8],
|
||||
c: TestData,
|
||||
d: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
enum TestData {
|
||||
Foo(FooStruct),
|
||||
Bar(BarStruct),
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
struct FooStruct {
|
||||
a: u64,
|
||||
b: [u8; 8],
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
struct BarStruct {
|
||||
a: u64,
|
||||
b: [u8; 8],
|
||||
}
|
||||
|
||||
impl TestStruct {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
a: 0x0102030405060708,
|
||||
b: [1, 2, 3, 4, 5, 6, 7, 8],
|
||||
c: TestData::Foo(FooStruct {
|
||||
a: 0x0102030405060708,
|
||||
b: [1, 2, 3, 4, 5, 6, 7, 8],
|
||||
}),
|
||||
d: String::from("Hello World!"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,70 +0,0 @@
|
||||
//! This module creates two sets of serialize and deserialize for bincode.
|
||||
//! They explicitly spell out the bincode settings so that switching to
|
||||
//! new versions in the future is less error prone.
|
||||
//!
|
||||
//! This is a list of all the row types and their settings for bincode.
|
||||
//! +--------------+--------+------------+----------------+------------+
|
||||
//! | | Endian | Int Length | Allow Trailing | Byte Limit |
|
||||
//! +--------------+--------+------------+----------------+------------+
|
||||
//! | TxHistoryRow | big | fixed | allow | unlimited |
|
||||
//! | All others | little | fixed | allow | unlimited |
|
||||
//! +--------------+--------+------------+----------------+------------+
|
||||
|
||||
// We only want people to use bincode_util
|
||||
use bincode::Options;
|
||||
use bincode_do_not_use_directly as bincode;
|
||||
|
||||
pub fn serialize_big<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
|
||||
where
|
||||
T: ?Sized + serde::Serialize,
|
||||
{
|
||||
big_endian().serialize(value)
|
||||
}
|
||||
|
||||
pub fn deserialize_big<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
|
||||
where
|
||||
T: serde::Deserialize<'a>,
|
||||
{
|
||||
big_endian().deserialize(bytes)
|
||||
}
|
||||
|
||||
pub fn serialize_little<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
|
||||
where
|
||||
T: ?Sized + serde::Serialize,
|
||||
{
|
||||
little_endian().serialize(value)
|
||||
}
|
||||
|
||||
pub fn deserialize_little<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
|
||||
where
|
||||
T: serde::Deserialize<'a>,
|
||||
{
|
||||
little_endian().deserialize(bytes)
|
||||
}
|
||||
|
||||
/// This is the default settings for Options,
|
||||
/// but all explicitly spelled out, except for endianness.
|
||||
/// The following functions will add endianness.
|
||||
#[inline]
|
||||
fn options() -> impl Options {
|
||||
bincode::options()
|
||||
.with_fixint_encoding()
|
||||
.with_no_limit()
|
||||
.allow_trailing_bytes()
|
||||
}
|
||||
|
||||
/// Adding the endian flag for big endian
|
||||
#[inline]
|
||||
fn big_endian() -> impl Options {
|
||||
options().with_big_endian()
|
||||
}
|
||||
|
||||
/// Adding the endian flag for little endian
|
||||
#[inline]
|
||||
fn little_endian() -> impl Options {
|
||||
options().with_little_endian()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "./bincode_tests.rs"]
|
||||
mod bincode_tests;
|
||||
@ -1,22 +1,21 @@
|
||||
use bitcoin::hashes::Hash;
|
||||
|
||||
use crate::chain::{BlockHash, BlockHeader};
|
||||
use crate::chain::{Block, BlockHeader};
|
||||
use crate::errors::*;
|
||||
use crate::new_index::BlockEntry;
|
||||
|
||||
use bitcoin::consensus::encode::serialize;
|
||||
use bitcoin::util::hash::BitcoinHash;
|
||||
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use std::iter::FromIterator;
|
||||
use std::slice;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
use time::OffsetDateTime as DateTime;
|
||||
|
||||
const MTP_SPAN: usize = 11;
|
||||
use time;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct BlockId {
|
||||
pub height: usize,
|
||||
pub hash: BlockHash,
|
||||
pub hash: Sha256dHash,
|
||||
pub time: u32,
|
||||
}
|
||||
|
||||
@ -33,12 +32,12 @@ impl From<&HeaderEntry> for BlockId {
|
||||
#[derive(Eq, PartialEq, Clone)]
|
||||
pub struct HeaderEntry {
|
||||
height: usize,
|
||||
hash: BlockHash,
|
||||
hash: Sha256dHash,
|
||||
header: BlockHeader,
|
||||
}
|
||||
|
||||
impl HeaderEntry {
|
||||
pub fn hash(&self) -> &BlockHash {
|
||||
pub fn hash(&self) -> &Sha256dHash {
|
||||
&self.hash
|
||||
}
|
||||
|
||||
@ -53,21 +52,23 @@ impl HeaderEntry {
|
||||
|
||||
impl fmt::Debug for HeaderEntry {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
let last_block_time = DateTime::from_unix_timestamp(self.header().time as i64).unwrap();
|
||||
let last_block_time = time::at_utc(time::Timespec::new(self.header().time as i64, 0))
|
||||
.rfc3339()
|
||||
.to_string();
|
||||
write!(
|
||||
f,
|
||||
"hash={} height={} @ {}",
|
||||
"best={} height={} @ {}",
|
||||
self.hash(),
|
||||
self.height(),
|
||||
last_block_time.format(&Rfc3339).unwrap(),
|
||||
last_block_time,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HeaderList {
|
||||
headers: Vec<HeaderEntry>,
|
||||
heights: HashMap<BlockHash, usize>,
|
||||
tip: BlockHash,
|
||||
heights: HashMap<Sha256dHash, usize>,
|
||||
tip: Sha256dHash,
|
||||
}
|
||||
|
||||
impl HeaderList {
|
||||
@ -75,57 +76,19 @@ impl HeaderList {
|
||||
HeaderList {
|
||||
headers: vec![],
|
||||
heights: HashMap::new(),
|
||||
tip: BlockHash::all_zeros(),
|
||||
tip: Sha256dHash::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new(
|
||||
mut headers_map: HashMap<BlockHash, BlockHeader>,
|
||||
tip_hash: BlockHash,
|
||||
) -> HeaderList {
|
||||
trace!(
|
||||
"processing {} headers, tip at {:?}",
|
||||
headers_map.len(),
|
||||
tip_hash
|
||||
);
|
||||
|
||||
let mut blockhash = tip_hash;
|
||||
let mut headers_chain: Vec<BlockHeader> = vec![];
|
||||
let null_hash = BlockHash::all_zeros();
|
||||
|
||||
while blockhash != null_hash {
|
||||
let header = headers_map.remove(&blockhash).unwrap_or_else(|| {
|
||||
panic!(
|
||||
"missing expected blockhash in headers map: {:?}, pointed from: {:?}",
|
||||
blockhash,
|
||||
headers_chain.last().map(|h| h.block_hash())
|
||||
)
|
||||
});
|
||||
blockhash = header.prev_blockhash;
|
||||
headers_chain.push(header);
|
||||
}
|
||||
headers_chain.reverse();
|
||||
|
||||
trace!(
|
||||
"{} chained headers ({} orphan blocks left)",
|
||||
headers_chain.len(),
|
||||
headers_map.len()
|
||||
);
|
||||
|
||||
let mut headers = HeaderList::empty();
|
||||
headers.apply(headers.order(headers_chain));
|
||||
headers
|
||||
}
|
||||
|
||||
pub fn order(&self, new_headers: Vec<BlockHeader>) -> Vec<HeaderEntry> {
|
||||
// header[i] -> header[i-1] (i.e. header.last() is the tip)
|
||||
struct HashedHeader {
|
||||
blockhash: BlockHash,
|
||||
blockhash: Sha256dHash,
|
||||
header: BlockHeader,
|
||||
}
|
||||
let hashed_headers =
|
||||
Vec::<HashedHeader>::from_iter(new_headers.into_iter().map(|header| HashedHeader {
|
||||
blockhash: header.block_hash(),
|
||||
blockhash: header.bitcoin_hash(),
|
||||
header,
|
||||
}));
|
||||
for i in 1..hashed_headers.len() {
|
||||
@ -138,31 +101,26 @@ impl HeaderList {
|
||||
Some(h) => h.header.prev_blockhash,
|
||||
None => return vec![], // hashed_headers is empty
|
||||
};
|
||||
let null_hash = BlockHash::all_zeros();
|
||||
let null_hash = Sha256dHash::default();
|
||||
let new_height: usize = if prev_blockhash == null_hash {
|
||||
0
|
||||
} else {
|
||||
self.header_by_blockhash(&prev_blockhash)
|
||||
.unwrap_or_else(|| panic!("{} is not part of the blockchain", prev_blockhash))
|
||||
.expect(&format!("{} is not part of the blockchain", prev_blockhash))
|
||||
.height()
|
||||
+ 1
|
||||
};
|
||||
(new_height..)
|
||||
.zip(hashed_headers)
|
||||
.zip(hashed_headers.into_iter())
|
||||
.map(|(height, hashed_header)| HeaderEntry {
|
||||
height,
|
||||
height: height,
|
||||
hash: hashed_header.blockhash,
|
||||
header: hashed_header.header,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Returns any rolled back blocks in order from old tip first and first block in the fork is last
|
||||
/// It also returns the blockhash of the post-rollback tip.
|
||||
pub fn apply(
|
||||
&mut self,
|
||||
new_headers: Vec<HeaderEntry>,
|
||||
) -> (Vec<HeaderEntry>, Option<BlockHash>) {
|
||||
pub fn apply(&mut self, new_headers: Vec<HeaderEntry>) {
|
||||
// new_headers[i] -> new_headers[i - 1] (i.e. new_headers.last() is the tip)
|
||||
for i in 1..new_headers.len() {
|
||||
assert_eq!(new_headers[i - 1].height() + 1, new_headers[i].height());
|
||||
@ -177,27 +135,19 @@ impl HeaderList {
|
||||
let expected_prev_blockhash = if height > 0 {
|
||||
*self.headers[height - 1].hash()
|
||||
} else {
|
||||
BlockHash::all_zeros()
|
||||
Sha256dHash::default()
|
||||
};
|
||||
assert_eq!(entry.header().prev_blockhash, expected_prev_blockhash);
|
||||
height
|
||||
}
|
||||
None => return (vec![], None),
|
||||
None => return,
|
||||
};
|
||||
debug!(
|
||||
"applying {} new headers from height {}",
|
||||
new_headers.len(),
|
||||
new_height
|
||||
);
|
||||
let mut removed = self.headers.split_off(new_height); // keep [0..new_height) entries
|
||||
|
||||
// If we reorged, we should return the last blockhash before adding the new chain's blockheaders.
|
||||
let reorged_tip = if !removed.is_empty() {
|
||||
self.headers.last().map(|be| be.hash()).cloned()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
self.headers.split_off(new_height); // keep [0..new_height) entries
|
||||
for new_header in new_headers {
|
||||
let height = new_header.height();
|
||||
assert_eq!(height, self.headers.len());
|
||||
@ -205,11 +155,9 @@ impl HeaderList {
|
||||
self.headers.push(new_header);
|
||||
self.heights.insert(self.tip, height);
|
||||
}
|
||||
removed.reverse();
|
||||
(removed, reorged_tip)
|
||||
}
|
||||
|
||||
pub fn header_by_blockhash(&self, blockhash: &BlockHash) -> Option<&HeaderEntry> {
|
||||
pub fn header_by_blockhash(&self, blockhash: &Sha256dHash) -> Option<&HeaderEntry> {
|
||||
let height = self.heights.get(blockhash)?;
|
||||
let header = self.headers.get(*height)?;
|
||||
if *blockhash == *header.hash() {
|
||||
@ -220,8 +168,9 @@ impl HeaderList {
|
||||
}
|
||||
|
||||
pub fn header_by_height(&self, height: usize) -> Option<&HeaderEntry> {
|
||||
self.headers.get(height).inspect(|entry| {
|
||||
self.headers.get(height).map(|entry| {
|
||||
assert_eq!(entry.height(), height);
|
||||
entry
|
||||
})
|
||||
}
|
||||
|
||||
@ -229,13 +178,13 @@ impl HeaderList {
|
||||
self.headers.last() == other.headers.last()
|
||||
}
|
||||
|
||||
pub fn tip(&self) -> &BlockHash {
|
||||
pub fn tip(&self) -> &Sha256dHash {
|
||||
assert_eq!(
|
||||
self.tip,
|
||||
self.headers
|
||||
.last()
|
||||
.map(|h| *h.hash())
|
||||
.unwrap_or(BlockHash::all_zeros())
|
||||
.unwrap_or(Sha256dHash::default())
|
||||
);
|
||||
&self.tip
|
||||
}
|
||||
@ -244,41 +193,20 @@ impl HeaderList {
|
||||
self.headers.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.headers.is_empty()
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> slice::Iter<'_, HeaderEntry> {
|
||||
pub fn iter(&self) -> slice::Iter<HeaderEntry> {
|
||||
self.headers.iter()
|
||||
}
|
||||
|
||||
/// Get the Median Time Past
|
||||
pub fn get_mtp(&self, height: usize) -> u32 {
|
||||
// Use the timestamp as the mtp of the genesis block.
|
||||
// Matches bitcoind's behaviour: bitcoin-cli getblock `bitcoin-cli getblockhash 0` | jq '.time == .mediantime'
|
||||
if height == 0 {
|
||||
self.headers.first().unwrap().header.time
|
||||
} else if height > self.len() - 1 {
|
||||
0
|
||||
} else {
|
||||
let mut timestamps = (height.saturating_sub(MTP_SPAN - 1)..=height)
|
||||
.map(|p_height| self.headers.get(p_height).unwrap().header.time)
|
||||
.collect::<Vec<_>>();
|
||||
timestamps.sort_unstable();
|
||||
timestamps[timestamps.len() / 2]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct BlockStatus {
|
||||
pub in_best_chain: bool,
|
||||
pub height: Option<usize>,
|
||||
pub next_best: Option<BlockHash>,
|
||||
pub next_best: Option<Sha256dHash>,
|
||||
}
|
||||
|
||||
impl BlockStatus {
|
||||
pub fn confirmed(height: usize, next_best: Option<BlockHash>) -> BlockStatus {
|
||||
pub fn confirmed(height: usize, next_best: Option<Sha256dHash>) -> BlockStatus {
|
||||
BlockStatus {
|
||||
in_best_chain: true,
|
||||
height: Some(height),
|
||||
@ -297,7 +225,6 @@ impl BlockStatus {
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct BlockMeta {
|
||||
#[serde(alias = "nTx")]
|
||||
pub tx_count: u32,
|
||||
pub size: u32,
|
||||
pub weight: u32,
|
||||
@ -306,18 +233,23 @@ pub struct BlockMeta {
|
||||
pub struct BlockHeaderMeta {
|
||||
pub header_entry: HeaderEntry,
|
||||
pub meta: BlockMeta,
|
||||
pub mtp: u32,
|
||||
}
|
||||
|
||||
impl From<&Block> for BlockMeta {
|
||||
fn from(block: &Block) -> BlockMeta {
|
||||
BlockMeta {
|
||||
tx_count: block.txdata.len() as u32,
|
||||
weight: block.txdata.iter().map(|tx| tx.get_weight() as u32).sum(),
|
||||
size: serialize(block).len() as u32,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&BlockEntry> for BlockMeta {
|
||||
fn from(b: &BlockEntry) -> BlockMeta {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
let weight = b.block.weight().to_wu() as u32;
|
||||
#[cfg(feature = "liquid")]
|
||||
let weight = b.block.weight() as u32;
|
||||
BlockMeta {
|
||||
tx_count: b.block.txdata.len() as u32,
|
||||
weight,
|
||||
weight: b.block.txdata.iter().map(|tx| tx.get_weight() as u32).sum(),
|
||||
size: b.size,
|
||||
}
|
||||
}
|
||||
|
||||
130
src/util/elements.rs
Normal file
130
src/util/elements.rs
Normal file
@ -0,0 +1,130 @@
|
||||
use bitcoin::blockdata::script::Instruction::PushBytes;
|
||||
use bitcoin::consensus::encode::serialize;
|
||||
use bitcoin::Script;
|
||||
use elements::confidential::Value;
|
||||
use elements::{AssetIssuance, Proof};
|
||||
|
||||
use hex;
|
||||
|
||||
use crate::chain::Network;
|
||||
use crate::util::{get_script_asm, script_to_address};
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct BlockProofValue {
|
||||
challenge: Script,
|
||||
challenge_asm: String,
|
||||
solution: Script,
|
||||
solution_asm: String,
|
||||
}
|
||||
|
||||
impl From<&Proof> for BlockProofValue {
|
||||
fn from(proof: &Proof) -> Self {
|
||||
BlockProofValue {
|
||||
challenge_asm: get_script_asm(&proof.challenge),
|
||||
challenge: proof.challenge.clone(),
|
||||
solution_asm: get_script_asm(&proof.solution),
|
||||
solution: proof.solution.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
pub struct IssuanceValue {
|
||||
pub is_reissuance: bool,
|
||||
pub asset_blinding_nonce: Option<String>,
|
||||
pub asset_entropy: Option<String>,
|
||||
pub assetamount: Option<u64>,
|
||||
pub assetamountcommitment: Option<String>,
|
||||
pub tokenamount: Option<u64>,
|
||||
pub tokenamountcommitment: Option<String>,
|
||||
}
|
||||
|
||||
impl From<&AssetIssuance> for IssuanceValue {
|
||||
fn from(issuance: &AssetIssuance) -> Self {
|
||||
let zero = [0u8; 32];
|
||||
let is_reissuance = issuance.asset_blinding_nonce != zero;
|
||||
|
||||
IssuanceValue {
|
||||
is_reissuance,
|
||||
asset_blinding_nonce: if is_reissuance {
|
||||
Some(hex::encode(issuance.asset_blinding_nonce))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
asset_entropy: if issuance.asset_entropy != zero {
|
||||
Some(hex::encode(issuance.asset_entropy))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
assetamount: match issuance.amount {
|
||||
Value::Explicit(value) => Some(value),
|
||||
_ => None,
|
||||
},
|
||||
assetamountcommitment: match issuance.amount {
|
||||
Value::Confidential(..) => Some(hex::encode(serialize(&issuance.amount))),
|
||||
_ => None,
|
||||
},
|
||||
tokenamount: match issuance.inflation_keys {
|
||||
Value::Explicit(value) => Some(value / 100000000), // https://github.com/ElementsProject/rust-elements/issues/7
|
||||
_ => None,
|
||||
},
|
||||
tokenamountcommitment: match issuance.inflation_keys {
|
||||
Value::Confidential(..) => Some(hex::encode(serialize(&issuance.inflation_keys))),
|
||||
_ => None,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
pub struct PegOutRequest {
|
||||
pub genesis_hash: String,
|
||||
pub scriptpubkey: Script,
|
||||
pub scriptpubkey_asm: String,
|
||||
pub scriptpubkey_address: Option<String>,
|
||||
}
|
||||
|
||||
impl PegOutRequest {
|
||||
pub fn parse(
|
||||
script: &Script,
|
||||
parent_network: &Network,
|
||||
parent_genesis_hash: &str,
|
||||
) -> Option<PegOutRequest> {
|
||||
if !script.is_op_return() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let nulldata: Vec<_> = script.iter(true).skip(1).collect();
|
||||
if nulldata.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let genesis_hash = if let PushBytes(data) = nulldata[0] {
|
||||
let mut data = data.to_vec();
|
||||
data.reverse();
|
||||
hex::encode(data)
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
|
||||
let scriptpubkey = if let PushBytes(data) = nulldata[1] {
|
||||
Script::from(data.to_vec())
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
|
||||
if genesis_hash != parent_genesis_hash {
|
||||
return None;
|
||||
}
|
||||
|
||||
let scriptpubkey_asm = get_script_asm(&scriptpubkey);
|
||||
let scriptpubkey_address = script_to_address(&scriptpubkey, parent_network);
|
||||
|
||||
Some(PegOutRequest {
|
||||
genesis_hash,
|
||||
scriptpubkey,
|
||||
scriptpubkey_asm,
|
||||
scriptpubkey_address,
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,4 +1,6 @@
|
||||
use crate::chain::{Network, Transaction, TxOut};
|
||||
#[cfg(feature = "liquid")]
|
||||
use crate::chain::Value;
|
||||
use crate::chain::{Transaction, TxOut};
|
||||
use std::collections::HashMap;
|
||||
|
||||
const VSIZE_BIN_WIDTH: u32 = 50_000; // in vbytes
|
||||
@ -10,12 +12,32 @@ pub struct TxFeeInfo {
|
||||
}
|
||||
|
||||
impl TxFeeInfo {
|
||||
pub fn new(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>, network: Network) -> Self {
|
||||
let fee = get_tx_fee(tx, prevouts, network);
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
let vsize = tx.weight().to_wu() / 4;
|
||||
#[cfg(feature = "liquid")]
|
||||
let vsize = tx.weight() / 4;
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
pub fn new(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>) -> Self {
|
||||
let total_in: u64 = prevouts.values().map(|prevout| prevout.value).sum();
|
||||
let total_out: u64 = tx.output.iter().map(|vout| vout.value).sum();
|
||||
let fee = total_in - total_out;
|
||||
let vsize = tx.get_weight() / 4;
|
||||
|
||||
TxFeeInfo {
|
||||
fee,
|
||||
vsize: vsize as u32,
|
||||
fee_per_vbyte: fee as f32 / vsize as f32,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn new(tx: &Transaction, _prevouts: &HashMap<u32, &TxOut>) -> Self {
|
||||
let fee = tx
|
||||
.output
|
||||
.iter()
|
||||
.find(|vout| vout.is_fee())
|
||||
.map_or(0, |vout| match vout.value {
|
||||
Value::Explicit(value) => value,
|
||||
_ => 0u64,
|
||||
});
|
||||
|
||||
let vsize = tx.get_weight() / 4;
|
||||
|
||||
TxFeeInfo {
|
||||
fee,
|
||||
@ -25,42 +47,23 @@ impl TxFeeInfo {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
pub fn get_tx_fee(tx: &Transaction, prevouts: &HashMap<u32, &TxOut>, _network: Network) -> u64 {
|
||||
if tx.is_coinbase() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let total_in: u64 = prevouts
|
||||
.values()
|
||||
.map(|prevout| prevout.value.to_sat())
|
||||
.sum();
|
||||
let total_out: u64 = tx.output.iter().map(|vout| vout.value.to_sat()).sum();
|
||||
total_in - total_out
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub fn get_tx_fee(tx: &Transaction, _prevouts: &HashMap<u32, &TxOut>, network: Network) -> u64 {
|
||||
tx.fee_in(*network.native_asset())
|
||||
}
|
||||
|
||||
pub fn make_fee_histogram(mut entries: Vec<&TxFeeInfo>) -> Vec<(f32, u32)> {
|
||||
entries.sort_unstable_by(|e1, e2| e1.fee_per_vbyte.partial_cmp(&e2.fee_per_vbyte).unwrap());
|
||||
|
||||
let mut histogram = vec![];
|
||||
let mut bin_size = 0;
|
||||
let mut last_fee_rate = 0.0;
|
||||
let mut last_fee_rate = None;
|
||||
for e in entries.iter().rev() {
|
||||
if bin_size > VSIZE_BIN_WIDTH && last_fee_rate != e.fee_per_vbyte {
|
||||
// vsize of transactions paying >= last_fee_rate
|
||||
histogram.push((last_fee_rate, bin_size));
|
||||
last_fee_rate = Some(e.fee_per_vbyte);
|
||||
bin_size += e.vsize;
|
||||
if bin_size > VSIZE_BIN_WIDTH {
|
||||
// vsize of transactions paying >= e.fee_per_vbyte
|
||||
histogram.push((e.fee_per_vbyte, bin_size));
|
||||
bin_size = 0;
|
||||
}
|
||||
last_fee_rate = e.fee_per_vbyte;
|
||||
bin_size += e.vsize;
|
||||
}
|
||||
if bin_size > 0 {
|
||||
histogram.push((last_fee_rate, bin_size));
|
||||
if let Some(fee_rate) = last_fee_rate {
|
||||
histogram.push((fee_rate, bin_size));
|
||||
}
|
||||
histogram
|
||||
}
|
||||
|
||||
@ -1,22 +1,20 @@
|
||||
use crate::chain::{BlockHash, Txid};
|
||||
use bitcoin_hashes::{sha256d::Hash as Sha256dHash, Hash};
|
||||
|
||||
use crate::errors::*;
|
||||
use crate::new_index::ChainQuery;
|
||||
use bitcoin::hashes::{sha256d::Hash as Sha256dHash, Hash};
|
||||
|
||||
pub fn get_tx_merkle_proof(
|
||||
chain: &ChainQuery,
|
||||
tx_hash: &Txid,
|
||||
block_hash: &BlockHash,
|
||||
tx_hash: &Sha256dHash,
|
||||
block_hash: &Sha256dHash,
|
||||
) -> Result<(Vec<Sha256dHash>, usize)> {
|
||||
let txids = chain
|
||||
.get_block_txids(block_hash)
|
||||
.get_block_txids(&block_hash)
|
||||
.chain_err(|| format!("missing block txids for #{}", block_hash))?;
|
||||
let pos = txids
|
||||
.iter()
|
||||
.position(|txid| txid == tx_hash)
|
||||
.chain_err(|| format!("missing txid {}", tx_hash))?;
|
||||
let txids = txids.into_iter().map(Sha256dHash::from).collect();
|
||||
|
||||
let (branch, _root) = create_merkle_branch_and_root(txids, pos);
|
||||
Ok((branch, pos))
|
||||
}
|
||||
@ -39,14 +37,12 @@ pub fn get_header_merkle_proof(
|
||||
);
|
||||
}
|
||||
|
||||
let heights: Vec<usize> = (0..=cp_height).collect();
|
||||
let header_hashes: Vec<BlockHash> = heights
|
||||
let heights: Vec<usize> = (0..cp_height + 1).collect();
|
||||
let header_hashes: Vec<Sha256dHash> = heights
|
||||
.into_iter()
|
||||
.map(|height| chain.hash_by_height(height))
|
||||
.collect::<Option<Vec<BlockHash>>>()
|
||||
.collect::<Option<Vec<Sha256dHash>>>()
|
||||
.chain_err(|| "missing block headers")?;
|
||||
|
||||
let header_hashes = header_hashes.into_iter().map(Sha256dHash::from).collect();
|
||||
Ok(create_merkle_branch_and_root(header_hashes, height))
|
||||
}
|
||||
|
||||
@ -55,7 +51,7 @@ pub fn get_id_from_pos(
|
||||
height: usize,
|
||||
tx_pos: usize,
|
||||
want_merkle: bool,
|
||||
) -> Result<(Txid, Vec<Sha256dHash>)> {
|
||||
) -> Result<(Sha256dHash, Vec<Sha256dHash>)> {
|
||||
let header_hash = chain
|
||||
.hash_by_height(height)
|
||||
.chain_err(|| format!("missing block #{}", height))?;
|
||||
@ -68,8 +64,6 @@ pub fn get_id_from_pos(
|
||||
.get(tx_pos)
|
||||
.chain_err(|| format!("No tx in position #{} in block #{}", tx_pos, height))?;
|
||||
|
||||
let txids = txids.into_iter().map(Sha256dHash::from).collect();
|
||||
|
||||
let branch = if want_merkle {
|
||||
create_merkle_branch_and_root(txids, tx_pos).0
|
||||
} else {
|
||||
@ -90,12 +84,12 @@ fn create_merkle_branch_and_root(
|
||||
let mut merkle = vec![];
|
||||
while hashes.len() > 1 {
|
||||
if hashes.len() % 2 != 0 {
|
||||
let last = *hashes.last().unwrap();
|
||||
let last = hashes.last().unwrap().clone();
|
||||
hashes.push(last);
|
||||
}
|
||||
index = if index % 2 == 0 { index + 1 } else { index - 1 };
|
||||
merkle.push(hashes[index]);
|
||||
index /= 2;
|
||||
index = index / 2;
|
||||
hashes = hashes
|
||||
.chunks(2)
|
||||
.map(|pair| merklize(pair[0], pair[1]))
|
||||
195
src/util/mod.rs
195
src/util/mod.rs
@ -1,31 +1,33 @@
|
||||
mod address;
|
||||
mod block;
|
||||
mod merkle;
|
||||
mod script;
|
||||
mod transaction;
|
||||
|
||||
pub mod bincode_util;
|
||||
pub mod electrum_merkle;
|
||||
pub mod fees;
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
mod elements;
|
||||
|
||||
pub use self::address::{Address, Payload};
|
||||
pub use self::block::{BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, HeaderEntry, HeaderList};
|
||||
pub use self::fees::get_tx_fee;
|
||||
pub use self::script::{
|
||||
get_innerscripts, IsProvablyUnspendable, ScriptToAddr, ScriptToAsm, SegwitDetection,
|
||||
};
|
||||
pub use self::transaction::{
|
||||
extract_tx_prevouts, has_prevout, is_coinbase, is_spendable, serialize_outpoint,
|
||||
sigops::transaction_sigop_count, TransactionStatus, TxInput,
|
||||
};
|
||||
pub use self::merkle::{get_header_merkle_proof, get_id_from_pos, get_tx_merkle_proof};
|
||||
pub use self::script::{get_script_asm, script_to_address};
|
||||
pub use self::transaction::{has_prevout, is_coinbase, is_spendable, TransactionStatus};
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub use self::elements::{BlockProofValue, IssuanceValue, PegOutRequest};
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
pub const REGTEST_INITIAL_ISSUANCE_PREVOUT: &str =
|
||||
"cb4953dfe64aba4687d0de08e5ce17d8f7640deeef63be4ac3bac3afdefdfd78";
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::mpsc::{channel, Receiver, Sender};
|
||||
use std::sync::Mutex;
|
||||
use std::thread::{self, ThreadId};
|
||||
use std::sync::mpsc::{channel, sync_channel, Receiver, Sender, SyncSender};
|
||||
use std::thread;
|
||||
|
||||
use crate::chain::BlockHeader;
|
||||
use bitcoin::hashes::sha256d::Hash as Sha256dHash;
|
||||
use socket2::{Domain, Protocol, Socket, Type};
|
||||
use std::net::SocketAddr;
|
||||
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
|
||||
|
||||
pub type Bytes = Vec<u8>;
|
||||
pub type HeaderMap = HashMap<Sha256dHash, BlockHeader>;
|
||||
@ -36,42 +38,30 @@ const HASH_LEN: usize = 32;
|
||||
pub type FullHash = [u8; HASH_LEN];
|
||||
|
||||
pub fn full_hash(hash: &[u8]) -> FullHash {
|
||||
*array_ref![hash, 0, HASH_LEN]
|
||||
array_ref![hash, 0, HASH_LEN].clone()
|
||||
}
|
||||
|
||||
pub struct SyncChannel<T> {
|
||||
tx: Option<crossbeam_channel::Sender<T>>,
|
||||
rx: Option<crossbeam_channel::Receiver<T>>,
|
||||
tx: SyncSender<T>,
|
||||
rx: Receiver<T>,
|
||||
}
|
||||
|
||||
impl<T> SyncChannel<T> {
|
||||
pub fn new(size: usize) -> SyncChannel<T> {
|
||||
let (tx, rx) = crossbeam_channel::bounded(size);
|
||||
SyncChannel {
|
||||
tx: Some(tx),
|
||||
rx: Some(rx),
|
||||
}
|
||||
let (tx, rx) = sync_channel(size);
|
||||
SyncChannel { tx, rx }
|
||||
}
|
||||
|
||||
pub fn sender(&self) -> crossbeam_channel::Sender<T> {
|
||||
self.tx.as_ref().expect("No Sender").clone()
|
||||
pub fn sender(&self) -> SyncSender<T> {
|
||||
self.tx.clone()
|
||||
}
|
||||
|
||||
pub fn receiver(&self) -> &crossbeam_channel::Receiver<T> {
|
||||
self.rx.as_ref().expect("No Receiver")
|
||||
pub fn receiver(&self) -> &Receiver<T> {
|
||||
&self.rx
|
||||
}
|
||||
|
||||
pub fn into_receiver(self) -> crossbeam_channel::Receiver<T> {
|
||||
self.rx.expect("No Receiver")
|
||||
}
|
||||
|
||||
/// This drops the sender and receiver, causing all other methods to panic.
|
||||
///
|
||||
/// Use only when you know that the channel will no longer be used.
|
||||
/// ie. shutdown.
|
||||
pub fn close(&mut self) -> Option<crossbeam_channel::Receiver<T>> {
|
||||
self.tx.take();
|
||||
self.rx.take()
|
||||
pub fn into_receiver(self) -> Receiver<T> {
|
||||
self.rx
|
||||
}
|
||||
}
|
||||
|
||||
@ -81,7 +71,7 @@ pub struct Channel<T> {
|
||||
}
|
||||
|
||||
impl<T> Channel<T> {
|
||||
pub fn unbounded() -> Self {
|
||||
pub fn new() -> Channel<T> {
|
||||
let (tx, rx) = channel();
|
||||
Channel { tx, rx }
|
||||
}
|
||||
@ -99,131 +89,14 @@ impl<T> Channel<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// This static HashMap contains all the threads spawned with [`spawn_thread`] with their name
|
||||
#[inline]
|
||||
pub fn with_spawned_threads<F>(f: F)
|
||||
where
|
||||
F: FnOnce(&mut HashMap<ThreadId, String>),
|
||||
{
|
||||
lazy_static! {
|
||||
static ref SPAWNED_THREADS: Mutex<HashMap<ThreadId, String>> = Mutex::new(HashMap::new());
|
||||
}
|
||||
let mut lock = match SPAWNED_THREADS.lock() {
|
||||
Ok(threads) => threads,
|
||||
// There's no possible broken state
|
||||
Err(threads) => {
|
||||
warn!("SPAWNED_THREADS is in a poisoned state! Be wary of incorrect logs!");
|
||||
threads.into_inner()
|
||||
}
|
||||
};
|
||||
f(&mut lock)
|
||||
}
|
||||
|
||||
pub fn spawn_thread<F, T>(prefix: &str, do_work: F) -> thread::JoinHandle<T>
|
||||
pub fn spawn_thread<F, T>(name: &str, f: F) -> thread::JoinHandle<T>
|
||||
where
|
||||
F: FnOnce() -> T,
|
||||
F: Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
static THREAD_COUNTER: AtomicUsize = AtomicUsize::new(0);
|
||||
let counter = THREAD_COUNTER.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
thread::Builder::new()
|
||||
.name(format!("{}-{}", prefix, counter))
|
||||
.spawn(move || {
|
||||
let thread = std::thread::current();
|
||||
let name = thread.name().unwrap();
|
||||
let id = thread.id();
|
||||
|
||||
trace!("[THREAD] GETHASHMAP INSERT | {name} {id:?}");
|
||||
with_spawned_threads(|threads| {
|
||||
threads.insert(id, name.to_owned());
|
||||
});
|
||||
trace!("[THREAD] START WORK | {name} {id:?}");
|
||||
|
||||
let result = do_work();
|
||||
|
||||
trace!("[THREAD] FINISHED WORK | {name} {id:?}");
|
||||
trace!("[THREAD] GETHASHMAP REMOVE | {name} {id:?}");
|
||||
with_spawned_threads(|threads| {
|
||||
threads.remove(&id);
|
||||
});
|
||||
trace!("[THREAD] HASHMAP REMOVED | {name} {id:?}");
|
||||
|
||||
result
|
||||
})
|
||||
.name(name.to_owned())
|
||||
.spawn(f)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
// Similar to https://doc.rust-lang.org/std/primitive.bool.html#method.then (nightly only),
|
||||
// but with a function that returns an `Option<T>` instead of `T`. Adding something like
|
||||
// this to std is being discussed: https://github.com/rust-lang/rust/issues/64260
|
||||
|
||||
pub trait BoolThen {
|
||||
fn and_then<T>(self, f: impl FnOnce() -> Option<T>) -> Option<T>;
|
||||
}
|
||||
|
||||
impl BoolThen for bool {
|
||||
fn and_then<T>(self, f: impl FnOnce() -> Option<T>) -> Option<T> {
|
||||
if self {
|
||||
f()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create_socket(addr: &SocketAddr) -> Socket {
|
||||
let domain = match &addr {
|
||||
SocketAddr::V4(_) => Domain::IPV4,
|
||||
SocketAddr::V6(_) => Domain::IPV6,
|
||||
};
|
||||
let socket =
|
||||
Socket::new(domain, Type::STREAM, Some(Protocol::TCP)).expect("creating socket failed");
|
||||
|
||||
#[cfg(unix)]
|
||||
socket
|
||||
.set_reuse_port(true)
|
||||
.expect("cannot enable SO_REUSEPORT");
|
||||
|
||||
socket.bind(&(*addr).into()).expect("cannot bind");
|
||||
|
||||
socket
|
||||
}
|
||||
|
||||
/// A module used for serde serialization of bytes in hexadecimal format.
|
||||
///
|
||||
/// The module is compatible with the serde attribute.
|
||||
///
|
||||
/// Copied from https://github.com/rust-bitcoin/rust-bitcoincore-rpc/blob/master/json/src/lib.rs
|
||||
pub mod serde_hex {
|
||||
use bitcoin::hashes::hex::FromHex;
|
||||
use serde::de::Error;
|
||||
use serde::{Deserializer, Serializer};
|
||||
|
||||
pub fn serialize<S: Serializer>(b: &[u8], s: S) -> Result<S::Ok, S::Error> {
|
||||
s.serialize_str(&hex::encode(b))
|
||||
}
|
||||
|
||||
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
|
||||
let hex_str: String = ::serde::Deserialize::deserialize(d)?;
|
||||
FromHex::from_hex(&hex_str).map_err(D::Error::custom)
|
||||
}
|
||||
|
||||
pub mod opt {
|
||||
use bitcoin::hashes::hex::FromHex;
|
||||
use serde::de::Error;
|
||||
use serde::{Deserializer, Serializer};
|
||||
|
||||
pub fn serialize<S: Serializer>(b: &Option<Vec<u8>>, s: S) -> Result<S::Ok, S::Error> {
|
||||
match *b {
|
||||
None => s.serialize_none(),
|
||||
Some(ref b) => s.serialize_str(&hex::encode(b)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Vec<u8>>, D::Error> {
|
||||
let hex_str: String = ::serde::Deserialize::deserialize(d)?;
|
||||
Ok(Some(FromHex::from_hex(&hex_str).map_err(D::Error::custom)?))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,196 +1,54 @@
|
||||
#[cfg(feature = "liquid")]
|
||||
use elements::address as elements_address;
|
||||
use crate::chain::Network;
|
||||
use crate::util::{Address, Payload};
|
||||
use bitcoin::Script;
|
||||
use bitcoin_bech32::constants::Network as B32Network;
|
||||
use bitcoin_bech32::{u5, WitnessProgram};
|
||||
use bitcoin_hashes::{hash160::Hash as Hash160, Hash};
|
||||
|
||||
use crate::chain::{script, Network, Script, TxIn, TxOut};
|
||||
use script::Instruction::PushBytes;
|
||||
|
||||
pub struct InnerScripts {
|
||||
pub redeem_script: Option<Script>,
|
||||
pub witness_script: Option<Script>,
|
||||
}
|
||||
|
||||
pub trait IsProvablyUnspendable {
|
||||
fn is_provably_unspendable_(&self) -> bool;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
impl IsProvablyUnspendable for bitcoin::Script {
|
||||
// is_provably_unspendable() is deprecated in rust-bitcoin
|
||||
// so we re-implement it here. Copy pasted.
|
||||
fn is_provably_unspendable_(&self) -> bool {
|
||||
use bitcoin::blockdata::opcodes::{
|
||||
Class::{IllegalOp, ReturnOp},
|
||||
ClassifyContext, Opcode,
|
||||
};
|
||||
|
||||
match self.as_bytes().first() {
|
||||
Some(b) => {
|
||||
let first = Opcode::from(*b);
|
||||
let class = first.classify(ClassifyContext::Legacy);
|
||||
|
||||
class == ReturnOp || class == IllegalOp
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
impl IsProvablyUnspendable for elements::Script {
|
||||
#[inline(always)]
|
||||
fn is_provably_unspendable_(&self) -> bool {
|
||||
// Not deprecated yet
|
||||
self.is_provably_unspendable()
|
||||
}
|
||||
}
|
||||
|
||||
// Extension trait for segwit script detection that works across bitcoin and elements
|
||||
pub trait SegwitDetection {
|
||||
fn segwit_is_p2wpkh(&self) -> bool;
|
||||
fn segwit_is_p2wsh(&self) -> bool;
|
||||
fn segwit_is_p2tr(&self) -> bool;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
impl SegwitDetection for bitcoin::Script {
|
||||
fn segwit_is_p2wpkh(&self) -> bool {
|
||||
self.is_p2wpkh()
|
||||
}
|
||||
fn segwit_is_p2wsh(&self) -> bool {
|
||||
self.is_p2wsh()
|
||||
}
|
||||
fn segwit_is_p2tr(&self) -> bool {
|
||||
self.is_p2tr()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
impl SegwitDetection for bitcoin::ScriptBuf {
|
||||
fn segwit_is_p2wpkh(&self) -> bool {
|
||||
self.is_p2wpkh()
|
||||
}
|
||||
fn segwit_is_p2wsh(&self) -> bool {
|
||||
self.is_p2wsh()
|
||||
}
|
||||
fn segwit_is_p2tr(&self) -> bool {
|
||||
self.is_p2tr()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
impl SegwitDetection for elements::Script {
|
||||
fn segwit_is_p2wpkh(&self) -> bool {
|
||||
self.is_v0_p2wpkh()
|
||||
}
|
||||
fn segwit_is_p2wsh(&self) -> bool {
|
||||
self.is_v0_p2wsh()
|
||||
}
|
||||
fn segwit_is_p2tr(&self) -> bool {
|
||||
self.is_v1_p2tr()
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ScriptToAsm: std::fmt::Debug {
|
||||
fn to_asm(&self) -> String {
|
||||
let asm = format!("{:?}", self);
|
||||
asm[7..asm.len() - 1].to_string()
|
||||
}
|
||||
}
|
||||
impl ScriptToAsm for bitcoin::Script {}
|
||||
impl ScriptToAsm for bitcoin::ScriptBuf {}
|
||||
#[cfg(feature = "liquid")]
|
||||
impl ScriptToAsm for elements::Script {}
|
||||
|
||||
pub trait ScriptToAddr {
|
||||
fn to_address_str(&self, network: Network) -> Option<String>;
|
||||
}
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
impl ScriptToAddr for bitcoin::Script {
|
||||
fn to_address_str(&self, network: Network) -> Option<String> {
|
||||
bitcoin::Address::from_script(self, bitcoin::Network::from(network))
|
||||
.ok()
|
||||
.map(|s| s.to_string())
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "liquid")]
|
||||
impl ScriptToAddr for elements::Script {
|
||||
fn to_address_str(&self, network: Network) -> Option<String> {
|
||||
elements_address::Address::from_script(self, None, network.address_params())
|
||||
.map(|a| a.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
// Returns the witnessScript in the case of p2wsh, or the redeemScript in the case of p2sh.
|
||||
pub fn get_innerscripts(txin: &TxIn, prevout: &TxOut) -> InnerScripts {
|
||||
// Wrapped redeemScript for P2SH spends
|
||||
let redeem_script = if prevout.script_pubkey.is_p2sh() {
|
||||
if let Some(Ok(PushBytes(redeemscript))) = txin.script_sig.instructions().last() {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
let bytes = redeemscript.as_bytes().to_vec();
|
||||
#[cfg(feature = "liquid")]
|
||||
let bytes = redeemscript.to_vec();
|
||||
Some(Script::from(bytes))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
// @XXX we can't use any of the Address:p2{...}h utility methods, since they expect the pre-image data, which we don't have.
|
||||
// we must instead create the Payload manually, which results in code duplication with the p2{...}h methods, especially for witness programs.
|
||||
// ideally, this should be implemented as part of the rust-bitcoin lib.
|
||||
pub fn script_to_address(script: &Script, network: &Network) -> Option<String> {
|
||||
let payload = if script.is_p2pkh() {
|
||||
Some(Payload::PubkeyHash(
|
||||
Hash160::from_slice(&script[3..23]).ok()?,
|
||||
))
|
||||
} else if script.is_p2sh() {
|
||||
Some(Payload::ScriptHash(
|
||||
Hash160::from_slice(&script[2..22]).ok()?,
|
||||
))
|
||||
} else if script.is_v0_p2wpkh() {
|
||||
Some(Payload::WitnessProgram(
|
||||
WitnessProgram::new(
|
||||
u5::try_from_u8(0).expect("0<32"),
|
||||
script[2..22].to_vec(),
|
||||
B32Network::from(network),
|
||||
)
|
||||
.unwrap(),
|
||||
))
|
||||
} else if script.is_v0_p2wsh() {
|
||||
Some(Payload::WitnessProgram(
|
||||
WitnessProgram::new(
|
||||
u5::try_from_u8(0).expect("0<32"),
|
||||
script[2..34].to_vec(),
|
||||
B32Network::from(network),
|
||||
)
|
||||
.unwrap(),
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Wrapped witnessScript for P2WSH or P2SH-P2WSH spends
|
||||
let witness_script = if prevout.script_pubkey.segwit_is_p2wsh()
|
||||
|| prevout.script_pubkey.segwit_is_p2tr()
|
||||
|| redeem_script.as_ref().is_some_and(|s| s.segwit_is_p2wsh())
|
||||
{
|
||||
let witness = &txin.witness;
|
||||
#[cfg(feature = "liquid")]
|
||||
let witness = &witness.script_witness;
|
||||
|
||||
// rust-bitcoin returns witness items as a [u8] slice, while rust-elements returns a Vec<u8>
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
let wit_to_vec = Vec::from;
|
||||
#[cfg(feature = "liquid")]
|
||||
let wit_to_vec = Clone::clone;
|
||||
|
||||
let inner_script_slice = if prevout.script_pubkey.segwit_is_p2tr() {
|
||||
// Witness stack is potentially very large
|
||||
// so we avoid to_vec() or iter().collect() for performance
|
||||
let w_len = witness.len();
|
||||
witness
|
||||
.last()
|
||||
// Get the position of the script spend script (if it exists)
|
||||
.map(|last_elem| {
|
||||
// From BIP341:
|
||||
// If there are at least two witness elements, and the first byte of
|
||||
// the last element is 0x50, this last element is called annex a
|
||||
// and is removed from the witness stack.
|
||||
if w_len >= 2 && last_elem.first().filter(|&&v| v == 0x50).is_some() {
|
||||
// account for the extra item removed from the end
|
||||
3
|
||||
} else {
|
||||
// otherwise script is 2nd from last
|
||||
2
|
||||
}
|
||||
})
|
||||
// Convert to None if not script spend
|
||||
// Note: Option doesn't have filter_map() method
|
||||
.filter(|&script_pos_from_last| w_len >= script_pos_from_last)
|
||||
.and_then(|script_pos_from_last| {
|
||||
// Can't use second_to_last() since it might be 3rd to last
|
||||
#[allow(clippy::iter_nth)]
|
||||
witness.iter().nth(w_len - script_pos_from_last)
|
||||
})
|
||||
} else {
|
||||
witness.last()
|
||||
};
|
||||
|
||||
inner_script_slice.map(wit_to_vec).map(Script::from)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
InnerScripts {
|
||||
redeem_script,
|
||||
witness_script,
|
||||
}
|
||||
Some(
|
||||
Address {
|
||||
payload: payload?,
|
||||
network: *network,
|
||||
}
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_script_asm(script: &Script) -> String {
|
||||
let asm = format!("{:?}", script);
|
||||
(&asm[7..asm.len() - 1]).to_string()
|
||||
}
|
||||
|
||||
@ -1,29 +1,18 @@
|
||||
use crate::chain::{BlockHash, OutPoint, Transaction, TxIn, TxOut, Txid};
|
||||
use crate::errors;
|
||||
use crate::util::{BlockId, IsProvablyUnspendable};
|
||||
use bitcoin_hashes::sha256d::Hash as Sha256dHash;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use crate::chain::{TxIn, TxOut};
|
||||
use crate::util::BlockId;
|
||||
|
||||
#[cfg(feature = "liquid")]
|
||||
lazy_static! {
|
||||
static ref REGTEST_INITIAL_ISSUANCE_PREVOUT: Txid =
|
||||
"50cdc410c9d0d61eeacc531f52d2c70af741da33af127c364e52ac1ee7c030a5"
|
||||
.parse()
|
||||
.unwrap();
|
||||
static ref TESTNET_INITIAL_ISSUANCE_PREVOUT: Txid =
|
||||
"0c52d2526a5c9f00e9fb74afd15dd3caaf17c823159a514f929ae25193a43a52"
|
||||
.parse()
|
||||
.unwrap();
|
||||
}
|
||||
use crate::util::REGTEST_INITIAL_ISSUANCE_PREVOUT;
|
||||
#[cfg(feature = "liquid")]
|
||||
use bitcoin_hashes::hex::ToHex;
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct TransactionStatus {
|
||||
pub confirmed: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub block_height: Option<usize>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub block_hash: Option<BlockHash>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub block_hash: Option<Sha256dHash>,
|
||||
pub block_time: Option<u32>,
|
||||
}
|
||||
|
||||
@ -32,7 +21,7 @@ impl From<Option<BlockId>> for TransactionStatus {
|
||||
match blockid {
|
||||
Some(b) => TransactionStatus {
|
||||
confirmed: true,
|
||||
block_height: Some(b.height),
|
||||
block_height: Some(b.height as usize),
|
||||
block_hash: Some(b.hash),
|
||||
block_time: Some(b.time),
|
||||
},
|
||||
@ -46,12 +35,6 @@ impl From<Option<BlockId>> for TransactionStatus {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct TxInput {
|
||||
pub txid: Txid,
|
||||
pub vin: u32,
|
||||
}
|
||||
|
||||
pub fn is_coinbase(txin: &TxIn) -> bool {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
return txin.previous_output.is_null();
|
||||
@ -65,328 +48,12 @@ pub fn has_prevout(txin: &TxIn) -> bool {
|
||||
#[cfg(feature = "liquid")]
|
||||
return !txin.is_coinbase()
|
||||
&& !txin.is_pegin
|
||||
&& txin.previous_output.txid != *REGTEST_INITIAL_ISSUANCE_PREVOUT
|
||||
&& txin.previous_output.txid != *TESTNET_INITIAL_ISSUANCE_PREVOUT;
|
||||
&& txin.previous_output.txid.to_hex() != REGTEST_INITIAL_ISSUANCE_PREVOUT;
|
||||
}
|
||||
|
||||
pub fn is_spendable(txout: &TxOut) -> bool {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
return !txout.script_pubkey.is_provably_unspendable_();
|
||||
return !txout.script_pubkey.is_provably_unspendable();
|
||||
#[cfg(feature = "liquid")]
|
||||
return !txout.is_fee() && !txout.script_pubkey.is_provably_unspendable_();
|
||||
}
|
||||
|
||||
/// Extract the previous TxOuts of a Transaction's TxIns
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// This function MUST NOT return an error variant when allow_missing is true.
|
||||
/// If allow_missing is false, it will return an error when any Outpoint is
|
||||
/// missing from the keys of the txos argument's HashMap.
|
||||
pub fn extract_tx_prevouts<'a>(
|
||||
tx: &Transaction,
|
||||
txos: &'a HashMap<OutPoint, TxOut>,
|
||||
) -> Result<HashMap<u32, &'a TxOut>, errors::Error> {
|
||||
tx.input
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, txi)| has_prevout(txi))
|
||||
.map(|(index, txi)| {
|
||||
Ok((
|
||||
index as u32,
|
||||
match txos.get(&txi.previous_output) {
|
||||
Some(txo) => txo,
|
||||
None => {
|
||||
return Err(format!("missing outpoint {:?}", txi.previous_output).into());
|
||||
}
|
||||
},
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn serialize_outpoint<S>(outpoint: &OutPoint, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::ser::Serializer,
|
||||
{
|
||||
use serde::ser::SerializeStruct;
|
||||
let mut s = serializer.serialize_struct("OutPoint", 2)?;
|
||||
s.serialize_field("txid", &outpoint.txid)?;
|
||||
s.serialize_field("vout", &outpoint.vout)?;
|
||||
s.end()
|
||||
}
|
||||
|
||||
pub(super) mod sigops {
|
||||
use crate::chain::{
|
||||
opcodes::all::{OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY},
|
||||
script::{self, Instruction},
|
||||
Transaction, TxOut, Witness,
|
||||
};
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
use bitcoin::opcodes::Opcode;
|
||||
#[cfg(feature = "liquid")]
|
||||
use elements::opcodes::All as Opcode;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Get sigop count for transaction. prevout_map must have all the prevouts.
|
||||
pub fn transaction_sigop_count(
|
||||
tx: &Transaction,
|
||||
prevout_map: &HashMap<u32, &TxOut>,
|
||||
) -> Result<usize, script::Error> {
|
||||
let input_count = tx.input.len();
|
||||
let mut prevouts = Vec::with_capacity(input_count);
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
let is_coinbase_or_pegin = tx.is_coinbase();
|
||||
#[cfg(feature = "liquid")]
|
||||
let is_coinbase_or_pegin = tx.is_coinbase() || tx.input.iter().any(|input| input.is_pegin);
|
||||
|
||||
if !is_coinbase_or_pegin {
|
||||
for idx in 0..input_count {
|
||||
prevouts.push(
|
||||
*prevout_map
|
||||
.get(&(idx as u32))
|
||||
.ok_or(script::Error::EarlyEndOfScript)?,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// coinbase tx won't use prevouts so it can be empty.
|
||||
get_sigop_cost(tx, &prevouts, true, true)
|
||||
}
|
||||
|
||||
fn decode_pushnum(op: &Opcode) -> Option<u8> {
|
||||
// 81 = OP_1, 96 = OP_16
|
||||
// 81 -> 1, so... 81 - 80 -> 1
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
let self_u8 = op.to_u8();
|
||||
#[cfg(feature = "liquid")]
|
||||
let self_u8 = op.into_u8();
|
||||
match self_u8 {
|
||||
81..=96 => Some(self_u8 - 80),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn count_sigops(script: &script::Script, accurate: bool) -> usize {
|
||||
let mut n = 0;
|
||||
let mut pushnum_cache = None;
|
||||
for inst in script.instructions() {
|
||||
match inst {
|
||||
Ok(Instruction::Op(opcode)) => {
|
||||
match opcode {
|
||||
OP_CHECKSIG | OP_CHECKSIGVERIFY => {
|
||||
n += 1;
|
||||
}
|
||||
OP_CHECKMULTISIG | OP_CHECKMULTISIGVERIFY => {
|
||||
match (accurate, pushnum_cache) {
|
||||
(true, Some(pushnum)) => {
|
||||
// Add the number of pubkeys in the multisig as sigop count
|
||||
n += usize::from(pushnum);
|
||||
}
|
||||
_ => {
|
||||
// MAX_PUBKEYS_PER_MULTISIG from Bitcoin Core
|
||||
// https://github.com/bitcoin/bitcoin/blob/v25.0/src/script/script.h#L29-L30
|
||||
n += 20;
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
pushnum_cache = decode_pushnum(&opcode);
|
||||
}
|
||||
}
|
||||
}
|
||||
// We ignore errors as well as pushdatas
|
||||
_ => {
|
||||
pushnum_cache = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
n
|
||||
}
|
||||
|
||||
/// Get the sigop count for legacy transactions
|
||||
fn get_legacy_sigop_count(tx: &Transaction) -> usize {
|
||||
let mut n = 0;
|
||||
for input in &tx.input {
|
||||
n += count_sigops(&input.script_sig, false);
|
||||
}
|
||||
for output in &tx.output {
|
||||
n += count_sigops(&output.script_pubkey, false);
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
fn get_p2sh_sigop_count(tx: &Transaction, previous_outputs: &[&TxOut]) -> usize {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
if tx.is_coinbase() {
|
||||
return 0;
|
||||
}
|
||||
#[cfg(feature = "liquid")]
|
||||
if tx.is_coinbase() {
|
||||
return 0;
|
||||
}
|
||||
let mut n = 0;
|
||||
for (input, prevout) in tx.input.iter().zip(previous_outputs.iter()) {
|
||||
if prevout.script_pubkey.is_p2sh() {
|
||||
if let Some(Ok(script::Instruction::PushBytes(redeem))) =
|
||||
input.script_sig.instructions().last()
|
||||
{
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
let script = script::Script::from_bytes(redeem.as_bytes());
|
||||
#[cfg(feature = "liquid")]
|
||||
let script = script::Script::from(redeem.to_vec());
|
||||
#[allow(clippy::needless_borrow)]
|
||||
{
|
||||
n += count_sigops(&script, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
fn get_witness_sigop_count(tx: &Transaction, previous_outputs: &[&TxOut]) -> usize {
|
||||
let mut n = 0;
|
||||
|
||||
#[inline]
|
||||
fn is_push_only(script: &script::Script) -> bool {
|
||||
for inst in script.instructions() {
|
||||
match inst {
|
||||
Err(_) => return false,
|
||||
Ok(Instruction::Op(_)) => return false,
|
||||
Ok(Instruction::PushBytes(_)) => {}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn last_pushdata(script: &script::Script) -> Option<&[u8]> {
|
||||
match script.instructions().last() {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
Some(Ok(Instruction::PushBytes(bytes))) => Some(bytes.as_bytes()),
|
||||
#[cfg(feature = "liquid")]
|
||||
Some(Ok(Instruction::PushBytes(bytes))) => Some(bytes),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn count_with_prevout(
|
||||
prevout: &TxOut,
|
||||
script_sig: &script::Script,
|
||||
witness: &Witness,
|
||||
) -> usize {
|
||||
let mut n = 0;
|
||||
|
||||
let script_owned;
|
||||
let script: &script::Script = if prevout.script_pubkey.is_witness_program() {
|
||||
&prevout.script_pubkey
|
||||
} else if prevout.script_pubkey.is_p2sh()
|
||||
&& is_push_only(script_sig)
|
||||
&& !script_sig.is_empty()
|
||||
{
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
{
|
||||
script_owned =
|
||||
script::ScriptBuf::from(last_pushdata(script_sig).unwrap().to_vec());
|
||||
}
|
||||
#[cfg(feature = "liquid")]
|
||||
{
|
||||
script_owned =
|
||||
script::Script::from(last_pushdata(script_sig).unwrap().to_vec());
|
||||
}
|
||||
&script_owned
|
||||
} else {
|
||||
return 0;
|
||||
};
|
||||
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
if script.is_p2wsh() {
|
||||
let bytes = script.as_bytes();
|
||||
n += sig_ops(witness, bytes[0], &bytes[2..]);
|
||||
} else if script.is_p2wpkh() {
|
||||
n += 1;
|
||||
}
|
||||
#[cfg(feature = "liquid")]
|
||||
if script.is_v0_p2wsh() {
|
||||
let bytes = script.as_bytes();
|
||||
n += sig_ops(witness, bytes[0], &bytes[2..]);
|
||||
} else if script.is_v0_p2wpkh() {
|
||||
n += 1;
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
for (input, prevout) in tx.input.iter().zip(previous_outputs.iter()) {
|
||||
n += count_with_prevout(prevout, &input.script_sig, &input.witness);
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
/// Get the sigop cost for this transaction.
|
||||
fn get_sigop_cost(
|
||||
tx: &Transaction,
|
||||
previous_outputs: &[&TxOut],
|
||||
verify_p2sh: bool,
|
||||
verify_witness: bool,
|
||||
) -> Result<usize, script::Error> {
|
||||
let mut n_sigop_cost = get_legacy_sigop_count(tx) * 4;
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
if tx.is_coinbase() {
|
||||
return Ok(n_sigop_cost);
|
||||
}
|
||||
#[cfg(feature = "liquid")]
|
||||
if tx.is_coinbase() || tx.input.iter().any(|input| input.is_pegin) {
|
||||
return Ok(n_sigop_cost);
|
||||
}
|
||||
if tx.input.len() != previous_outputs.len() {
|
||||
return Err(script::Error::EarlyEndOfScript);
|
||||
}
|
||||
if verify_witness && !verify_p2sh {
|
||||
return Err(script::Error::EarlyEndOfScript);
|
||||
}
|
||||
if verify_p2sh {
|
||||
n_sigop_cost += get_p2sh_sigop_count(tx, previous_outputs) * 4;
|
||||
}
|
||||
if verify_witness {
|
||||
n_sigop_cost += get_witness_sigop_count(tx, previous_outputs);
|
||||
}
|
||||
|
||||
Ok(n_sigop_cost)
|
||||
}
|
||||
|
||||
/// Get sigops for the Witness
|
||||
///
|
||||
/// witness_version is the raw opcode. OP_0 is 0, OP_1 is 81, etc.
|
||||
#[allow(clippy::redundant_closure)]
|
||||
fn sig_ops(witness: &Witness, witness_version: u8, witness_program: &[u8]) -> usize {
|
||||
#[cfg(feature = "liquid")]
|
||||
let last_witness = witness.script_witness.last();
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
let last_witness = witness.last();
|
||||
match (witness_version, witness_program.len()) {
|
||||
(0, 20) => 1,
|
||||
(0, 32) => {
|
||||
#[cfg(not(feature = "liquid"))]
|
||||
{
|
||||
#[allow(clippy::needless_borrow)]
|
||||
last_witness
|
||||
.map(|sl| script::Script::from_bytes(sl))
|
||||
.map(|s| count_sigops(s, true))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
#[cfg(feature = "liquid")]
|
||||
{
|
||||
last_witness
|
||||
.map(|sl| script::Script::from(sl.clone()))
|
||||
.map(|s| count_sigops(&s, true))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
}
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
return !txout.is_fee() && !txout.script_pubkey.is_provably_unspendable();
|
||||
}
|
||||
|
||||
274
start
274
start
@ -1,274 +0,0 @@
|
||||
#!/usr/bin/env zsh
|
||||
|
||||
# initialize variables
|
||||
DAEMON=bitcoin
|
||||
NETWORK=mainnet
|
||||
FEATURES=default
|
||||
DB_FOLDER=/electrs
|
||||
ASSET_DB_ARGS=()
|
||||
NODENAME=$(hostname|cut -d . -f1)
|
||||
LOCATION=$(hostname|cut -d . -f2)
|
||||
USAGE="Usage: $0 (mainnet|testnet|signet|liquid|liquidtestnet) [popular-scripts]"
|
||||
|
||||
# load rust if necessary
|
||||
if [ -e "${HOME}/.cargo/env" ];then
|
||||
source "${HOME}/.cargo/env"
|
||||
export PATH="${HOME}/.cargo/bin:${PATH}"
|
||||
fi
|
||||
|
||||
# which OS?
|
||||
case "$(uname -s)" in
|
||||
FreeBSD)
|
||||
OS=FreeBSD
|
||||
NPROC=$(sysctl -n hw.ncpu)
|
||||
export CC=/usr/local/bin/clang17
|
||||
export CXX=/usr/local/bin/clang++17
|
||||
export CPP=/usr/local/bin/clang-cpp17
|
||||
export RUSTFLAGS="-C linker=clang17"
|
||||
;;
|
||||
Darwin)
|
||||
OS=Darwin
|
||||
NPROC=$(sysctl -n hw.ncpu)
|
||||
;;
|
||||
Linux)
|
||||
OS=Linux
|
||||
NPROC=$(grep -c proc /proc/cpuinfo)
|
||||
;;
|
||||
*)
|
||||
OS=Unknown
|
||||
NPROC=4
|
||||
;;
|
||||
esac
|
||||
|
||||
# which network?
|
||||
case "${1}" in
|
||||
mainnet)
|
||||
THREADS=$((NPROC / 8))
|
||||
CRONJOB_TIMING="20 4 * * *"
|
||||
;;
|
||||
testnet)
|
||||
NETWORK=testnet
|
||||
THREADS=$((NPROC / 8))
|
||||
CRONJOB_TIMING="2 4 * * *"
|
||||
;;
|
||||
testnet4)
|
||||
NETWORK=testnet4
|
||||
THREADS=$((NPROC / 8))
|
||||
CRONJOB_TIMING="17 4 * * *"
|
||||
;;
|
||||
signet)
|
||||
NETWORK=signet
|
||||
THREADS=$((NPROC / 8))
|
||||
CRONJOB_TIMING="9 4 * * *"
|
||||
;;
|
||||
liquid)
|
||||
DAEMON=elements
|
||||
NETWORK=liquid
|
||||
FEATURES=liquid
|
||||
ASSET_DB_ARGS=(--asset-db-path /elements/asset_registry_db)
|
||||
THREADS=$((NPROC / 8))
|
||||
CRONJOB_TIMING="12 4 * * *"
|
||||
;;
|
||||
liquidtestnet)
|
||||
DAEMON=elements
|
||||
NETWORK=liquidtestnet
|
||||
FEATURES=liquid
|
||||
ASSET_DB_ARGS=(--asset-db-path /elements/asset_registry_testnet_db)
|
||||
THREADS=$((NPROC / 8))
|
||||
CRONJOB_TIMING="17 4 * * *"
|
||||
;;
|
||||
*)
|
||||
echo "${USAGE}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Run the popular address txt file generator before each run
|
||||
POPULAR_SCRIPTS_FOLDER="${HOME}/popular-scripts/${NETWORK}"
|
||||
POPULAR_SCRIPTS_FILE_RAW="${POPULAR_SCRIPTS_FOLDER}/popular-scripts-raw.txt"
|
||||
POPULAR_SCRIPTS_FILE="${POPULAR_SCRIPTS_FOLDER}/popular-scripts.txt"
|
||||
|
||||
# This function runs the job for generating the popular scripts text file for the precache arg
|
||||
generate_popular_scripts() {
|
||||
mkdir -p "${POPULAR_SCRIPTS_FOLDER}"
|
||||
|
||||
## Use nproc * 4 threads to generate the txt file (lots of iowait, so 2x~4x core count is ok)
|
||||
## Only pick up addresses with 101 history events or more
|
||||
## (Without lowering MIN_HISTORY_ITEMS_TO_CACHE this is the lowest we can go)
|
||||
## It prints out progress to STDERR
|
||||
echo "[*] Generating popular-scripts using ${THREADS} threads..."
|
||||
cd "${HOME}/electrs"
|
||||
HIGH_USAGE_THRESHOLD=101 \
|
||||
JOB_THREAD_COUNT=${THREADS} \
|
||||
nice cargo run \
|
||||
--release \
|
||||
--bin popular-scripts \
|
||||
--features "${FEATURES}" \
|
||||
-- \
|
||||
--network "${NETWORK}" \
|
||||
--db-dir "${DB_FOLDER}" \
|
||||
> "${POPULAR_SCRIPTS_FILE_RAW}"
|
||||
|
||||
## Only overwrite the existing file if the popular-scripts cargo run succeeded
|
||||
if [ "$?" = "0" ];then
|
||||
## Sorted and deduplicated just in case
|
||||
echo "Sorting popular scripts for final results..."
|
||||
sort "${POPULAR_SCRIPTS_FILE_RAW}" | uniq > "${POPULAR_SCRIPTS_FILE}"
|
||||
fi
|
||||
|
||||
rm "${POPULAR_SCRIPTS_FILE_RAW}"
|
||||
}
|
||||
|
||||
# This function is for inserting the cronjob for generating the popular scripts
|
||||
CRONJOB_CMD="\"${HOME}/electrs/start\" \"${NETWORK}\" popular-scripts"
|
||||
echo "${CRONJOB_TIMING} ${CRONJOB_CMD}"
|
||||
|
||||
case "${2}" in
|
||||
popular-scripts)
|
||||
echo "[*] Only generate popular-scripts, then exit"
|
||||
generate_popular_scripts
|
||||
exit 0
|
||||
;;
|
||||
version)
|
||||
echo "[*] Only print versions, then exit"
|
||||
cargo run --bin electrs --release -- --version
|
||||
cargo run --bin popular-scripts --release -- --version
|
||||
exit 0
|
||||
;;
|
||||
"")
|
||||
# If the 2nd arg isn't passed, just run the normal electrs script as-is
|
||||
;;
|
||||
*)
|
||||
echo "${USAGE}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# run in loop in case of crash
|
||||
until false
|
||||
do
|
||||
# reset CWD
|
||||
cd "${HOME}/electrs"
|
||||
|
||||
# disable making electrs.core files
|
||||
ulimit -c 0
|
||||
|
||||
# prepare run-time variables
|
||||
UTXOS_LIMIT=500
|
||||
ELECTRUM_TXS_LIMIT=500
|
||||
ELECTRUM_MAX_LINE_SIZE=1048576 # 1 MiB
|
||||
ELECTRUM_MAX_SUBSCRIPTIONS=100
|
||||
ELECTRUM_MAX_CLIENTS=1000
|
||||
MAIN_LOOP_DELAY=500
|
||||
DAEMON_CONF="${HOME}/${DAEMON}.conf"
|
||||
HTTP_SOCKET_FILE="${HOME}/socket/esplora-${DAEMON}-${NETWORK}"
|
||||
RPC_SOCKET_FILE="${HOME}/socket/electrum-${DAEMON}-${NETWORK}"
|
||||
|
||||
# get RPC credentials from bitcoin.conf or elements.conf directly
|
||||
echo "[*] Getting RPC credentials from ${DAEMON_CONF}"
|
||||
RPC_USER=$(grep 'rpcuser=' "${DAEMON_CONF}"|cut -d = -f2|head -1)
|
||||
RPC_PASS=$(grep 'rpcpassword=' "${DAEMON_CONF}"|cut -d = -f2|head -1)
|
||||
|
||||
# override limits based on hostname
|
||||
if [ "${NODENAME}" = "node201" ];then
|
||||
UTXOS_LIMIT=9000
|
||||
ELECTRUM_TXS_LIMIT=9000
|
||||
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
|
||||
ELECTRUM_MAX_SUBSCRIPTIONS=100000
|
||||
ELECTRUM_MAX_CLIENTS=10000
|
||||
MAIN_LOOP_DELAY=14000
|
||||
fi
|
||||
if [ "${NODENAME}" = "node204" ] && [ "${LOCATION}" = "sg1" ];then
|
||||
UTXOS_LIMIT=9000
|
||||
ELECTRUM_TXS_LIMIT=9000
|
||||
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
|
||||
ELECTRUM_MAX_SUBSCRIPTIONS=100000
|
||||
ELECTRUM_MAX_CLIENTS=10000
|
||||
fi
|
||||
if [ "${NODENAME}" = "node204" ] && [ "${LOCATION}" = "hnl" ];then
|
||||
UTXOS_LIMIT=9000
|
||||
ELECTRUM_TXS_LIMIT=9000
|
||||
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
|
||||
ELECTRUM_MAX_SUBSCRIPTIONS=100000
|
||||
ELECTRUM_MAX_CLIENTS=10000
|
||||
fi
|
||||
if [ "${NODENAME}" = "node206" ] && [ "${LOCATION}" = "tk7" ];then
|
||||
UTXOS_LIMIT=9000
|
||||
ELECTRUM_TXS_LIMIT=9000
|
||||
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
|
||||
ELECTRUM_MAX_SUBSCRIPTIONS=100000
|
||||
ELECTRUM_MAX_CLIENTS=10000
|
||||
fi
|
||||
if [ "${NODENAME}" = "node211" ];then
|
||||
UTXOS_LIMIT=9000
|
||||
ELECTRUM_TXS_LIMIT=9000
|
||||
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
|
||||
ELECTRUM_MAX_SUBSCRIPTIONS=100000
|
||||
ELECTRUM_MAX_CLIENTS=10000
|
||||
fi
|
||||
if [ "${NODENAME}" = "node212" ];then
|
||||
UTXOS_LIMIT=9000
|
||||
ELECTRUM_TXS_LIMIT=9000
|
||||
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
|
||||
ELECTRUM_MAX_SUBSCRIPTIONS=100000
|
||||
ELECTRUM_MAX_CLIENTS=10000
|
||||
fi
|
||||
if [ "${NODENAME}" = "node213" ];then
|
||||
UTXOS_LIMIT=9000
|
||||
ELECTRUM_TXS_LIMIT=9000
|
||||
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
|
||||
ELECTRUM_MAX_SUBSCRIPTIONS=100000
|
||||
ELECTRUM_MAX_CLIENTS=10000
|
||||
fi
|
||||
if [ "${NODENAME}" = "node214" ];then
|
||||
UTXOS_LIMIT=9000
|
||||
ELECTRUM_TXS_LIMIT=9000
|
||||
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
|
||||
ELECTRUM_MAX_SUBSCRIPTIONS=100000
|
||||
ELECTRUM_MAX_CLIENTS=10000
|
||||
fi
|
||||
if [ "${NETWORK}" = "testnet4" ];then
|
||||
UTXOS_LIMIT=9000
|
||||
ELECTRUM_TXS_LIMIT=9000
|
||||
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
|
||||
ELECTRUM_MAX_SUBSCRIPTIONS=100000
|
||||
ELECTRUM_MAX_CLIENTS=10000
|
||||
fi
|
||||
if [ "${LOCATION}" = "fmt" ];then
|
||||
UTXOS_LIMIT=9000
|
||||
ELECTRUM_TXS_LIMIT=9000
|
||||
ELECTRUM_MAX_LINE_SIZE=16777216 # 16 MiB
|
||||
ELECTRUM_MAX_SUBSCRIPTIONS=100000
|
||||
ELECTRUM_MAX_CLIENTS=10000
|
||||
fi
|
||||
|
||||
if [ ! -e "${POPULAR_SCRIPTS_FILE}" ];then
|
||||
generate_popular_scripts
|
||||
fi
|
||||
|
||||
# Run the electrs process (Note: db-dir is used in both commands)
|
||||
nice cargo run \
|
||||
--release \
|
||||
--bin electrs \
|
||||
--features "${FEATURES}" \
|
||||
-- \
|
||||
--network "${NETWORK}" \
|
||||
--daemon-dir "${HOME}" \
|
||||
--db-dir "${DB_FOLDER}" \
|
||||
"${ASSET_DB_ARGS[@]}" \
|
||||
--main-loop-delay "${MAIN_LOOP_DELAY}" \
|
||||
--rpc-socket-file "${RPC_SOCKET_FILE}" \
|
||||
--http-socket-file "${HTTP_SOCKET_FILE}" \
|
||||
--precache-scripts "${POPULAR_SCRIPTS_FILE}" \
|
||||
--precache-threads "${THREADS}" \
|
||||
--cookie "${RPC_USER}:${RPC_PASS}" \
|
||||
--cors '*' \
|
||||
--address-search \
|
||||
--utxos-limit "${UTXOS_LIMIT}" \
|
||||
--electrum-txs-limit "${ELECTRUM_TXS_LIMIT}" \
|
||||
--electrum-max-line-size "${ELECTRUM_MAX_LINE_SIZE}" \
|
||||
--electrum-max-subscriptions "${ELECTRUM_MAX_SUBSCRIPTIONS}" \
|
||||
--electrum-max-clients "${ELECTRUM_MAX_CLIENTS}" \
|
||||
-vv
|
||||
sleep 1
|
||||
done
|
||||
Loading…
Reference in New Issue
Block a user