add privacy-preserving hourly aggregate of historical scan stats

.github/workflows/package.yaml

@@ -10,31 +10,45 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-2022, ubuntu-22.04, ubuntu-22.04-arm, macos-13, macos-14]
+        os: [windows-2022, ubuntu-22.04, ubuntu-22.04-arm, macos-15-intel, macos-14]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
-          submodules: true
-      - name: Set up JDK 22.0.2
-        uses: actions/setup-java@v4
+          submodules: recursive
+      - name: Clear Java tool-cache for reproducibility
+        shell: bash
+        run: rm -rf "$RUNNER_TOOL_CACHE"/Java_*
+      - name: Set up JDK 25.0.2
+        uses: actions/setup-java@v5
         with:
           distribution: 'temurin'
-          java-version: '22.0.2'
+          java-version: '25.0.2'
       - name: Show Build Versions
         run: ./gradlew -v
       - name: Build with Gradle
         run: ./gradlew jpackage
-      - name: Package zip distribution
-        if: ${{ runner.os == 'Windows' || runner.os == 'macOS' }}
+      - name: Codesign, package and notarize macOS distribution
+        if: ${{ runner.os == 'macOS' }}
+        uses: sparrowwallet/github-actions/codesign-macos@v1
+        with:
+          app-name: Frigate
+          certificate: ${{ secrets.MACOS_CERTIFICATE }}
+          certificate-password: ${{ secrets.MACOS_CERTIFICATE_PASSWORD }}
+          apple-id: ${{ secrets.MACOS_NOTARIZATION_APPLE_ID }}
+          team-id: ${{ secrets.MACOS_NOTARIZATION_TEAM_ID }}
+          notarization-password: ${{ secrets.MACOS_NOTARIZATION_PASSWORD }}
+      - name: Package Windows zip distribution
+        if: ${{ runner.os == 'Windows' }}
         run: ./gradlew packageZipDistribution
-      - name: Package tar distribution
+      - name: Package Linux tar distribution
         if: ${{ runner.os == 'Linux' }}
         run: ./gradlew packageTarDistribution
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: Frigate Build - ${{ runner.os }} ${{ runner.arch }}
           path: |
             build/jpackage/*
             !build/jpackage/Frigate/
             !build/jpackage/frigate/
+            !build/jpackage/Frigate.app/

.sdkmanrc

@@ -0,0 +1,3 @@
+# Enable auto-env through the sdkman_auto_env config
+# Add key=value pairs of SDKs to use below
+java=25.0.2-tem

README.md

@@ -1,280 +1,152 @@
 ![Frigate logo](https://github.com/sparrowwallet/frigate/raw/refs/heads/master/frigatelogo.png)
 
 # Frigate Electrum Server
- 
-Frigate is an experimental Electrum Server testing Silent Payments scanning with ephemeral client keys.
 
-It has three goals:
-1. To provide a proof of concept implementation of the [Remote Scanner](https://github.com/silent-payments/BIP0352-index-server-specification/blob/main/README.md#remote-scanner-ephemeral) approach discussed in the BIP352 Silent Payments Index Server [Specification](https://github.com/silent-payments/BIP0352-index-server-specification/blob/main/README.md) (WIP).
-2. To propose Electrum RPC protocol methods to request and return Silent Payments information from a server.
-3. To demonstrate an efficient "in database" technique of scanning for Silent Payments transactions.
+Frigate is an Electrum server for [Silent Payments](https://github.com/bitcoin/bips/blob/master/bip-0352.mediawiki) (BIP352).
+It performs Silent Payments scanning server-side using ephemeral client keys, returning discovered transactions to clients over an extension to the Electrum JSON-RPC protocol.
+Scanning runs in-database with optional GPU acceleration, supporting both single-user instances on commodity hardware and multi-user public servers.
 
-#### This is alpha software, and should not be used in production.
+## Features
 
-## Motivation
+- Server-side Silent Payments scanning with ephemeral (in-RAM) client keys, following the [Remote Scanner](https://github.com/silent-payments/BIP0352-index-server-specification/blob/main/README.md#remote-scanner-ephemeral) approach in the BIP352 Index Server Specification.
+- GPU-accelerated scanning with CUDA (NVIDIA), Metal (Apple), and OpenCL (Intel/AMD/NVIDIA) backends, selected automatically at runtime.
+- In-database EC point computation via a custom [DuckDB extension](https://github.com/sparrowwallet/duckdb-ufsecp-extension) wrapping [UltrafastSecp256k1](https://github.com/shrec/UltrafastSecp256k1).
+- Low-latency mempool ingestion via Bitcoin Core's ZMQ `sequence` publisher.
 
-[BIP 352](https://github.com/bitcoin/bips/blob/master/bip-0352.mediawiki) has proposed that light clients use compact block filters to scan for UTXOs received to a Silent Payments address.
-However, this introduces two significant problems:
+## Quick Start
 
-The first is one of data gravity.
-For any reasonable scan period, the client must download gigabytes of data in tweaks, block filters and finally some of the blocks themselves.
-All this data needs to be downloaded, parsed and potentially saved to avoid downloading it again, requiring significant resources on the client. 
-A client would likely need several gigabytes of data to restore a wallet with historical transactions, which is resource intensive in terms of bandwidth, CPU and storage. 
-Compare this to current Electrum clients which may use just a few megabytes to restore a wallet, and it's easy to see how this approach is unlikely to see widespread adoption - it's just too onerous, particularly for mobile clients.
+1. Install Bitcoin Core 28 or higher with `txindex=1` in `bitcoin.conf`. (28+ is required for Electrum protocol 1.6; earlier versions work for 1.3–1.5.)
+2. Download the latest Frigate release for your platform from the [Releases](https://github.com/sparrowwallet/frigate/releases) page (`.deb` / `.rpm` for Linux, `.dmg` for macOS, `.zip` for Windows). Releases are signed — see [GPG Key](#gpg-key).
+3. Start Frigate:
+   ```shell
+   /opt/frigate/bin/frigate                            # Linux .deb / .rpm
+   /Applications/Frigate.app/Contents/MacOS/Frigate    # macOS
+   ```
+4. With Bitcoin Core on the same host using default settings, no further configuration is required. Frigate creates `~/.frigate/config.toml` on first start.
+5. Frigate indexes from Taproot activation (mainnet) up to the chain tip before serving Silent Payments queries. Watch the log for `Electrum server listening on tcp://...` for the readiness signal.
+6. (Optional) Verify with the bundled CLI:
+   ```shell
+   /opt/frigate/bin/frigate-cli
+   ```
 
-The second problem is the lack of mempool monitoring, which is not supported with compact block filters. 
-Users rely on mempool monitoring to answer the "did you get my transaction?" question.
-The lack of ability to do this can cause user confusion and distrust in the wallet, which education can only go some way in reducing.
-
-This project attempts to address these problems using an Electrum protocol styled approach.
-Instead of asking the client to download the required data and perform the scanning, the server performs the scanning locally with an optimized index.
-This is the [Remote Scanner](https://github.com/silent-payments/BIP0352-index-server-specification/blob/main/README.md#remote-scanner-ephemeral) approach discussed in the BIP352 Silent Payments Index Server [Specification](https://github.com/silent-payments/BIP0352-index-server-specification/blob/main/README.md) (WIP).
-It should be noted that both the scan private key and the spend public key must be provided to the server in this approach.
-While this does have a privacy implication, the keys are not stored and only held by the server ephemerally (in RAM) for duration of the client session.
-This is similar to the widely used public Electrum server approach, where the wallet addresses are shared ephemerally with a public server. 
-
-## Approach
-
-The key problem that BIP 352 introduces with respect to scanning is that much of the computation cannot be done generally ahead of time.
-Instead, for every silent payment address, each transaction in the blockchain must be considered separately to determine if it sends funds to that address.
-The computation involves several cryptographic operations, including two resource intensive EC point multiplication operations on _every_ eligible transaction.
-In order to ensure that client keys are ephemeral and not stored, this computation must be done in a reasonable period of time on millions of transactions.
-
-This is the key difference between Silent Payments wallets and traditional BIP32 wallets, which can rely on a simple monotonically incrementing derivation path index.
-While Silent Payments provides important advantages in privacy and user experience, this computational burden is the downside that cannot be avoided.
-Any solution addressing the retrieval of Silent Payments transactions will eventually be bounded by the performance of EC point multiplication.
-For best performance and user experience this should be done as efficiently as possible, and therefore as close the source data as possible.
-
-In order to achieve this, Frigate addresses the problem of data gravity directly.
-Like most light client silent payment services, it builds an index of the data that can be pre-computed, generally known as a tweak index.
-This index contains a large number of elements (one for every silent payments eligible transaction in the blockchain) containing a tweak calculated from the public keys used in the transaction inputs.
-Frigate stores this data in a single table with the following schema:
-
-| Column       | Type         |
-|--------------|--------------|
-| `txid`       | BLOB         |
-| `height`     | INTEGER      |
-| `tweak_key`  | BLOB         |
-| `outputs`    | LIST(BIGINT) |
-
-The `txid` and `tweak_key` values are 32 and 33 byte BLOBS respectively. 
-The `outputs` value is a list of 8 byte integers, each representing the first 8 bytes of the x-value of the Taproot output public key.
-
-On startup, Frigate connects to the configured Bitcoin Core RPC, downloads blocks from the configured block height (or from Taproot activation on mainnet) and adds entries to the table.
-Once it has reached the blockchain tip, it starts a simple (and incomplete) Electrum Server to interface with the client.
-
-The scanning is the interesting part.
-Instead of loading data from the table into the Frigate server application, the database itself performs all the required cryptographic operations. 
-To do this, Frigate uses a fast OLAP database called [DuckDB](https://duckdb.org/why_duckdb.html#fast) designed for analytical query workloads.
-It then extends the database with [a custom extension](https://github.com/sparrowwallet/duckdb-secp256k1-extension) that adds functions from [libsecp256k1](https://github.com/bitcoin-core/secp256k1).
-This allows Frigate to perform functions such as
-```sql
-SELECT secp256k1_ec_pubkey_tweak_mul(tweak_key, scalar);
-```
-which allows the EC point computation to happen as close to the tweak data as possible.
-
-With these extensions, Frigate performs a query as follows:
-```sql
-SELECT txid, height FROM tweak WHERE list_contains(outputs, hash_prefix_to_int(secp256k1_ec_pubkey_combine([SPEND_PUBLIC_KEY, secp256k1_ec_pubkey_create(secp256k1_tagged_sha256('BIP0352/SharedSecret', secp256k1_ec_pubkey_tweak_mul(tweak_key, SCAN_PRIVATE_KEY) || int_to_big_endian(0)))]), 1));
-```
-This computes the Taproot output key for `k = 0` and compares it to the list of known keys for each tweak row, returning the `txid` and `height` if there is a match.
-The client can then download the transaction and determine if it does indeed contain outputs it is interested in, including for higher values of `k`.
-
-## Electrum protocol
-
-The Electrum protocol is by far the most widely used light client protocol for Bitcoin wallets, and support is now almost a requirement for widespread adoption of any wallet technology proposal.
-It is characterised by resource efficiency for the client in terms of bandwidth, CPU and storage, allowing a good user experience on almost any platform. 
-It has however been designed around BIP32 wallets. 
-Silent Payments presents an alternative model, where instead of an incrementing derivation path index (and associated gap limit) transactions must be found through scanning the blockchain.
-As such, new methods are necessary.
-Frigate proposes the following Electrum JSON-RPC methods:
-
-### blockchain.silentpayments.subscribe
-
-**Signature**
-```
-blockchain.silentpayments.subscribe(scan_private_key, spend_public_key, start)
-```
-
-- _scan_private_key_: A 64 character string containing the hex of the scan private key.
-- _spend_public_key_: A 66 character string containing the hex of the spend public key.
-- _start_: (Optional) Block height or timestamp to start scanning from. Values above 500,000,000 are treated as seconds from the start of the epoch.
-
-**Result**
-
-The silent payment address that has been subscribed.
-
-**Result Example**
-
-```
-sp1qqgste7k9hx0qftg6qmwlkqtwuy6cycyavzmzj85c6qdfhjdpdjtdgqjuexzk6murw56suy3e0rd2cgqvycxttddwsvgxe2usfpxumr70xc9pkqwv
-```
-
-### Notifications
-
-Once subscribed, the client will receive notifications as results are returned from the scan with the following signature. 
-All historical (`progress` < `1.0`) results **must** be sent before current (up to date) results.
-Once the client has received a notification with `progress` == `1.0`, it should consider the scan complete.
-
-```
-blockchain.silentpayments.subscribe(subscription, progress, history)
-```
-
-**Result**
-
-A dictionary with the following key/value pairs:
-
-1. A `subscription` JSON object literal containing details of the current subscription:
-- _address_: The silent payment address that has been subscribed to.
-- _start_height_: The block height from which the subscription scan was started.
-
-2. A `progress` key/value pair indicating the progress of a historical scan:
-- _progress_: A floating point value between `0.0` and `1.0`. Will be `1.0` for all current (up to date) results.
-
-3. A `history` array of transactions. Confirmed transactions are listed in blockchain order. Each transaction is a dictionary with the following keys:
-- _height_: The integer height of the block the transaction was confirmed in. For mempool transactions, `0` should be used.
-- _tx_hash_: The transaction hash in hexadecimal.
-
-**Result Example**
-
-```json
-{
-  "subscription": {
-    "address": "sp1qqgste7k9hx0qftg6qmwlkqtwuy6cycyavzmzj85c6qdfhjdpdjtdgqjuexzk6murw56suy3e0rd2cgqvycxttddwsvgxe2usfpxumr70xc9pkqwv",
-    "start_height": 882000
-  },
-  "progress": 1.0,
-  "history": [
-    {
-      "height": 890004,
-      "tx_hash": "acc3758bd2a26f869fcc67d48ff30b96464d476bca82c1cd6656e7d506816412"
-    },
-    {
-      "height": 905008,
-      "tx_hash": "f3e1bf48975b8d6060a9de8884296abb80be618dc00ae3cb2f6cee3085e09403"
-    },
-    {
-      "height": 0,
-      "tx_hash": "f4184fc596403b9d638783cf57adfe4c75c605f6356fbc91338530e9831e9e16"
-    }
-  ]
-}
-```
-
-It is recommended that servers implementing this protocol send history results incrementally as the historical scan progresses.
-In addition, a maximum page size of 100 is suggested when sending historical transactions.
-This will avoid transmission issues with large wallets that have many transactions, while providing the client with regular progress updates.
-In the case of block reorgs, the server should rescan all existing subscriptions from the reorg-ed block height and send any history (if found) to the client.
-All found mempool transactions should be sent on the initial subscription, but thereafter previously sent mempool transactions should not be resent.
-
-Clients should retrieve the transactions listed in the history with `blockchain.transaction.get` and subscribe to all owned outputs with `blockchain.scripthash.subscribe`. 
-Electrum wallet functionality then proceeds as normal.
-In other words, the silent payments address subscription is a replacement for the monotonically increasing derivation path index in BIP32 wallets.
-The subscription seeks only to add to the client's knowledge of incoming silent payments transactions.
-The client is responsible for checking the transactions do actually send to addresses it has keys for, and using normal Electrum wallet synchronization techniques to monitor for changes to these addresses. 
-
-### blockchain.silentpayments.unsubscribe
-
-**Signature**
-```
-blockchain.silentpayments.unsubscribe(scan_private_key, spend_public_key)
-```
-
-- _scan_private_key_: A 64 character string containing the hex of the scan private key.
-- _spend_public_key_: A 66 character string containing the hex of the spend public key.
-
-**Result**
-
-The silent payment address that has been unsubscribed. This should cancel any scans that may be currently running for this address.
-
-**Result Example**
-
-```
-sp1qqgste7k9hx0qftg6qmwlkqtwuy6cycyavzmzj85c6qdfhjdpdjtdgqjuexzk6murw56suy3e0rd2cgqvycxttddwsvgxe2usfpxumr70xc9pkqwv
-```
-
-## Performance
-
-The scanning query is essentially CPU bound, mostly around EC point multiplication.
-[DuckDB parallelizes](https://duckdb.org/docs/stable/guides/performance/how_to_tune_workloads#parallelism-multi-core-processing) the workload based on row groups, with each row group containing 122,880 rows.
-It will by default configure itself to use all the available cores on the server it is running.
-The behaviour can be configured in the Frigate configuration file (see `dbThreads`).
-
-The following set of benchmarks was generated on a M1 Macbook Pro with 10 available CPUs, scanning mainnet to a block height of 911434 with a database size of ~13Gb.
-**Note that no cut-through or dust filter has been used.**
-
-|                       |   Blocks  |   Start   | Transactions |   Time       | Transactions/sec |
-|-----------------------|-----------|-----------|--------------|--------------|------------------|
-|   2 hours             |   12      |   911422  | 8961         |   474ms      | 18905            |
-|   1 day               |   144     |   911290  | 149059       |   5s 6ms     | 29776            |
-|   1 week              |   1008    |   910426  | 1143906      |   7s 992ms   | 143131           |
-|   2 weeks             |   2016    |   909418  | 2349028      |   17s 408ms  | 134940           |
-|   4 weeks             |   4032    |   907402  | 5002030      |   36s 796ms  | 135940           |
-|   8 weeks             |   8064    |   903370  | 9441899      |   1m 6s      | 142101           |
-|   16 weeks            |   16128   |   895306  | 15910877     |   1m 51s     | 143269           |
-|   32 weeks            |   32256   |   879178  | 32666940     |   3m 47s     | 143638           |
-|   64 weeks            |   64512   |   846922  | 77427166     |   8m 55s     | 144606           |
-|   Taproot Activation  |   201802  |   709632  | 153651412    |   17m 25s    | 147043           |
-
-Higher performance on the longer periods is possible by increasing the number of CPUs.
-The following set of benchmarks was generated on an Intel server with 32 cores using the same tweak database:
-
-|                    |   Blocks  |   Start   | Transactions | Time      | Transactions/sec |
-|--------------------|-----------|-----------|--------------|-----------|------------------|
-| 2 hours            |   12      |   911422  | 8961         | 1s 345ms  | 6662             |
-| 1 day              |   144     |   911290  | 149059       | 7s 703ms  | 19351            |
-| 1 week             |   1008    |   910426  | 1143906      | 9s 625ms  | 118847           |
-| 2 weeks            |   2016    |   909418  | 2349028      | 14s 714ms | 159646           |
-| 4 weeks            |   4032    |   907402  | 5002030      | 27s 666ms | 180801           |
-| 8 weeks            |   8064    |   903370  | 9441899      | 44s 979ms | 209918           |
-| 16 weeks           |   16128   |   895306  | 15910877     | 1m 20s    | 199695           |
-| 32 weeks           |   32256   |   879178  | 32666940     | 2m 30s    | 217561           |
-| 64 weeks           |   64512   |   846922  | 77427166     | 5m 45s    | 224315           |
-| Taproot Activation |   201802  |   709632  | 153651412    | 11m 34s   | 221502           |
-
-Multiple clients conducting simultaneous scans slows each scan linearly. 
-Further performance improvements (or handling additional clients) may be performed by scaling out across [multiple read-only replicas of the database](https://motherduck.com/docs/key-tasks/authenticating-and-connecting-to-motherduck/read-scaling/).
-It is also possible to consider hardware acceleration techniques such as [HSMs](https://docs.aws.amazon.com/cloudhsm/latest/userguide/performance.html), [cryptographic coprocessors](https://developer.arm.com/Processors/CryptoCell-310) or GPU acceleration.
+For non-mainnet networks, pass `-n testnet|testnet4|signet|regtest`.
 
 ## Configuration
 
-For indexing Frigate will need access to the Bitcoin Core RPC, which will need to have `txindex=1` configured.
+Frigate stores its configuration in `~/.frigate/config.toml` on macOS and Linux, and `%APPDATA%\Frigate\config.toml` on Windows.
+A default configuration file is created on first startup. For indexing, Frigate needs access to the Bitcoin Core RPC, which requires `txindex=1` in `bitcoin.conf`.
 
-By default Frigate stores all configuration in `~/.frigate/config` on macOS and Linux, and `%APPDATA%/Frigate` on Windows.
-An example configuration looks as follows
-```json
-{
-  "coreServer": "http://127.0.0.1:8332",
-  "coreAuthType": "COOKIE",
-  "coreDataDir": "/home/bitcoin/.bitcoin",
-  "coreAuth": "bitcoin:password",
-  "startIndexing": true,
-  "indexStartHeight": 0,
-  "scriptPubKeyCacheSize": 10000000
-}
+With Bitcoin Core running on the same machine with default settings, Frigate will connect automatically with no configuration changes required.
+
+```toml
+# Frigate configuration
+
+[core]
+connect = true
+# server = "http://127.0.0.1:8332"
+# authType = "COOKIE"            # COOKIE or USERPASS
+# dataDir = "/home/bitcoin/.bitcoin"
+# auth = "user:password"         # only needed for USERPASS
+# zmqSequenceEndpoint = "tcp://127.0.0.1:28336"   # bitcoind -zmqpubsequence endpoint for low-latency mempool ingestion
+# rpcRequestTimeoutSeconds = 60                   # per-RPC read timeout (raise on slow/remote bitcoind, lower on fast LAN)
+# rpcBatchSize = 100                              # max sub-requests per JSON-RPC array batch (mempool fill)
+
+[index]
+# startHeight = 0                # default: 709632 on mainnet (Taproot activation), 0 on testnet
+# cacheSize = "10M"              # scriptPubKey cache entries (default: 10M, ~4GB RAM)
+
+[scan]
+# batchSize = 300000             # rows per GPU dispatch (reduce if scanning hangs on older GPUs)
+# computeBackend = "AUTO"        # AUTO, GPU, or CPU
+# dbThreads = 4                  # limit DuckDB threads (reduces CPU load when computeBackend = "CPU")
+# memoryLimit = "8GB"            # cap DuckDB memory usage (default: 80% of system RAM)
+# maxLabels = 10                 # maximum number of labels accepted per silent payments subscription
+# maxSubscriptions = 100         # maximum number of silent payments subscriptions per connection
+# metricsEnabled = true          # hourly aggregate scan stats log line (default: true)
+
+[server]
+# host = "ssl://xyz.com:50002"   # advertised in server.features; use array for multiple. Omit to advertise nothing.
+# tcp = "tcp://0.0.0.0:50001"    # plaintext listener bind URL; omit (or "") to disable. Default if neither tcp nor ssl is set.
+# ssl = "ssl://0.0.0.0:50002"    # SSL listener bind URL; omit to disable
+# sslCert = "cert.pem"           # PEM certificate (chain allowed); bare filename resolves under Frigate's home dir, or use an absolute path
+# sslKey  = "key.pem"            # PEM-encoded PKCS#8 private key; bare filename resolves under Frigate's home dir, or use an absolute path
+# backendElectrumServer = "tcp://localhost:60001"   # backend must listen on a port distinct from Frigate's tcp/ssl listeners above
 ```
 
-Default values for these entries will be set on first startup.
-The value of `coreAuthType` can either be `COOKIE` or `USERPASS`. 
-Configure `coreDataDir` or `coreAuth` respectively to grant RPC access.
-The value of `startIndexing` can be set to `false` if an index has already been built and you just want to execute queries against it without connecting to Bitcoin Core.
+### Core
+
+Set `connect = false` to run Frigate without connecting to Bitcoin Core.
+This is useful if an index has already been built and you just want to serve queries against it.
+The `authType` can be `COOKIE` (default) or `USERPASS`.
+For cookie authentication, set `dataDir` to the Bitcoin Core data directory if it is not in the default location.
+For user/password authentication, set `auth` to `user:password`.
+
+When `zmqSequenceEndpoint` is set, Frigate subscribes to Bitcoin Core's ZMQ `sequence` publisher for low-latency mempool ingestion (sub-100ms instead of up to 5s).
+This requires Bitcoin Core to be started with `-zmqpubsequence=tcp://...:<port>` matching this endpoint, and a Bitcoin Core build with ZMQ support compiled in.
+Release binaries from bitcoincore.org include ZMQ support; a from-source build needs the ZeroMQ development library (e.g. `libzmq3-dev`) and an explicit `-DWITH_ZMQ=ON`, since CMake silently disables ZMQ if the library is not found.
+Verify support with `bitcoin-cli getzmqnotifications` — a `-32601 Method not found` error means the binary has no ZMQ support.
+
+**Configuring `zmqSequenceEndpoint` is strongly recommended whenever `backendElectrumServer` is configured.**
+Without this configured, the backend may notify the client of a new transaction via scripthash before Frigate's silent payments notification lands — causing wallets to briefly display incorrect amounts.
+
+The `rpcRequestTimeoutSeconds` setting controls the per-RPC read timeout against Bitcoin Core (default: 60).
+Raise it for a slow or remote node if you see read timeouts in the log; the default is fine for a co-located Bitcoin Core.
+The `rpcBatchSize` setting caps the number of sub-requests per JSON-RPC array batch used during initial mempool indexing (default: 100); the default suits most deployments.
+
+### Index
 
 Indexing speed is greatly affected by looking up the scriptPubKeys of spent outputs.
 To improve performance, scriptPubKeys are cached to avoid looking them up again with `getrawtransaction`.
-The `scriptPubKeyCacheSize` limits the number of scriptPubKeys cached during indexing. 
-The default value leads to a total application memory size of around 4Gb. 
-This value can be increased or decreased depending on available RAM. 
+The `cacheSize` limits the number of scriptPubKeys cached during indexing (e.g. `"10M"` for 10 million entries, ~4GB RAM).
+This value can be increased or decreased depending on available RAM.
 
 The DuckDB database is stored in a `db` subfolder in the same directory, in a file called `frigate.duckdb`.
 DuckDB databases can be transferred between different operating systems, and should survive unclean shutdowns.
 
-To reduce CPU load while scanning, add an entry to reduce the number of cores made available to DuckDB, for example:
-```json
-{
-  "dbThreads": 2
-}
+### Scan
+
+The `computeBackend` setting controls whether historical scanning uses GPU or CPU. Valid values are `AUTO` (default), `GPU`, and `CPU`.
+In `AUTO` mode, the GPU is used if one is detected, otherwise the CPU is used.
+Set to `CPU` to force CPU-only scanning.
+With CPU-only scanning, `dbThreads` can be used to limit the number of DuckDB threads and reduce CPU load.
+The `memoryLimit` setting caps DuckDB's memory usage (e.g. `"8GB"`, `"1024MB"`). DuckDB's default is 80% of system RAM.
+Mempool and incremental block scans always run on the CPU backend, since they are short, latency-sensitive and benefit from being decoupled from the longer-running historical scans.
+
+The `batchSize` setting controls how many transactions are processed per GPU dispatch (default: 300,000).
+If scanning hangs or becomes unstable on certain GPUs (particularly older OpenCL-only GPUs), try reducing this value (e.g. 10,000 to 50,000).
+
+The `maxLabels` setting caps the number of labels accepted per silent payments subscription (default: 10).
+The `maxSubscriptions` setting caps the number of silent payments subscriptions per connection (default: 100).
+Requests exceeding either limit are rejected with a JSON-RPC `-32602 Invalid params` error.
+
+When `metricsEnabled` is true (default), Frigate emits one `Aggregate SP scan stats` log line per hour summarising historical scan throughput across all scans in the window.
+The output is bucketed by result count and duration, rounded, and suppresses any bucket with fewer than ten samples, so no per-client scan information is exposed. Set to `false` to disable the line entirely.
+
+### Server
+
+Listeners are configured as bind URLs in the scheme `tcp://` or `ssl://` followed by a host and port.
+The `tcp` setting is the plaintext listener (default: `tcp://0.0.0.0:50001`), and `ssl` enables a TLS listener (e.g. `ssl://0.0.0.0:50002`).
+Each URL specifies the bind interface — use `127.0.0.1` to restrict a listener to localhost, or `0.0.0.0` to bind all interfaces.
+Both listeners can run simultaneously, or either can be omitted (e.g. set `tcp = ""` to run TLS only).
+If neither `tcp` nor `ssl` is set, Frigate defaults to tcp on `0.0.0.0:50001`.
+
+To enable TLS, set `ssl` and supply a certificate and private key via `sslCert` and `sslKey`.
+Bare filenames are resolved relative to Frigate's home directory (the per-network directory that holds `config.toml`). Absolute paths are used as-is.
+The certificate file may contain a single certificate or a full chain (`fullchain.pem`), and the key file must be an unencrypted PKCS#8 (`-----BEGIN PRIVATE KEY-----`) PEM.
+TLS 1.0, 1.1 and SSLv3 are unconditionally disabled as insecure, and only TLS 1.2 and 1.3 are negotiated.
+
+To generate a self-signed certificate for local or testing use, valid for ten years:
+```shell
+openssl req -x509 -newkey rsa:2048 -keyout ~/.frigate/key.pem -out ~/.frigate/cert.pem -days 3650 -nodes -subj "/CN=localhost"
 ```
 
+Frigate implements the Silent Payments RPCs natively and proxies all other Electrum requests (including address-related lookups) to a co-located Electrum backend.
+The backend is configured with `backendElectrumServer`, and is intended to point to a server running on the same host.
+Because Frigate occupies the canonical 50001/50002 Electrum ports, a co-located backend (Fulcrum, ElectrumX, electrs, etc.) must be configured to listen on a different port.
+The Electrum protocol from 1.3 to 1.6 is supported — for 1.6, ensure Bitcoin Core 28 or higher.
+
+When `backendElectrumServer` is set, also configure `zmqSequenceEndpoint` under `[core]` — see the [Core](#core) section above for why this pairing matters.
+
 ## Usage
 
 The Frigate server may be started as follows:
@@ -292,6 +164,11 @@ To start with a different network, use the `-n` parameter:
 ./bin/frigate -n signet
 ```
 
+To change the home directory for config and data storage (default: `~/.frigate/` on Linux/macOS, `%APPDATA%\Frigate` on Windows), use the `-d` parameter:
+```shell
+./bin/frigate -d /var/lib/frigate
+```
+
 The full range of options can be queried with:
 ```shell
 ./bin/frigate -h
@@ -337,6 +214,497 @@ The full range of options can be queried with:
 ./bin/frigate-cli -h
 ```
 
+## Deployment
+
+### Recommended Topology
+
+The expected production deployment is a single host running Bitcoin Core, Frigate, and a co-located Electrum backend (Fulcrum, electrs, or ElectrumX):
+
+```
+                 ┌─────────────────────────────────────────────┐
+client ─TLS/TCP─►│  Frigate :50001 / :50002                    │
+                 │    │                                        │
+                 │    ├─ Silent Payments RPCs (native)         │
+                 │    └─ all other RPCs ─► Electrum backend    │
+                 │                          :60001 (localhost) │
+                 │                            │                │
+                 │                            ▼                │
+                 │                         Bitcoin Core        │
+                 │                          :8332 RPC          │
+                 │                          :28336 ZMQ         │
+                 └─────────────────────────────────────────────┘
+```
+
+Frigate occupies the canonical Electrum ports (50001/50002). 
+The backend listens on a non-conflicting port (e.g. 60001) bound to localhost, and all non-silent-payments queries are proxied to it transparently.
+
+### Backend Limits
+
+All Frigate client connections to `backendElectrumServer` appear from a single source IP, so backend per-IP caps (e.g. Fulcrum's `max_clients_per_ip`) can refuse new Frigate clients well below the backend's overall capacity.
+The co-located localhost topology above is handled by default: Fulcrum exempts `127.0.0.1/32` and `::1/128`, and ElectrumX exempts all private addresses (RFC1918, loopback, link-local).
+If Frigate runs on a different host from the backend, on Fulcrum add Frigate's IP to `subnets_to_exclude_from_per_ip_limits`.
+On ElectrumX the exemption set is hardcoded to private addresses, so keep Frigate on the same private network.
+electrs imposes no per-IP limits and requires no tuning.
+
+### Running as a Service
+
+Frigate runs as a long-lived foreground process and integrates straightforwardly with `systemd`. A minimal unit (`/etc/systemd/system/frigate.service`):
+
+```ini
+[Unit]
+Description=Frigate Electrum Server
+After=network-online.target bitcoind.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=frigate
+StateDirectory=frigate
+ExecStart=/opt/frigate/bin/frigate -d /var/lib/frigate
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+```
+
+`StateDirectory=frigate` ensures `/var/lib/frigate` exists and is owned by the service user, so the dedicated `frigate` account does not need a real home directory (`useradd --system --no-create-home frigate`). Adjust `User=` and the binary path to match your install.
+
+### Public-facing TLS
+
+For public deployments, supply a real certificate via `sslCert` and `sslKey` rather than the self-signed example in [Configuration > Server](#server).
+The `host` field under `[server]` is advertised in `server.features` and should match the hostname clients use when validating the certificate.
+TLS termination at a reverse proxy is also supported – in that case, run Frigate with `tcp = "tcp://127.0.0.1:50001"` and terminate TLS upstream.
+
+### Resource Requirements
+
+- **RAM**: indexing uses ~4 GB for the default scriptPubKey cache (`cacheSize = "10M"`). Serving steady-state queries is dominated by DuckDB's memory pool, which defaults to 80% of system RAM and can be capped via `memoryLimit`. 16 GB is a comfortable baseline; 8 GB works with a reduced cache and memory limit.
+- **Disk**: ~18 GB for the DuckDB index, growing slowly over time.
+- **CPU**: any modern x86-64 or ARM64 server. Indexing is I/O- and RPC-bound rather than CPU-bound.
+- **GPU**: optional but strongly recommended for multi-user instances. A single discrete GPU (NVIDIA Ampere or newer, Apple Silicon, or any OpenCL 1.2+ device including integrated GPUs) is sufficient for several simultaneous scans. See [GPU Requirements](#gpu-requirements).
+
+## Operations
+
+### Initial Sync
+
+On first start Frigate indexes from Taproot activation (block 709,632 on mainnet) up to the chain tip. 
+It does not begin serving Silent Payments queries until the index reaches the tip — at that point it logs `Electrum server listening on tcp://...`, which is the readiness signal. 
+Indexing time depends on RPC throughput from Bitcoin Core.
+
+### Logs
+
+Logs are written to `frigate.log` in the per-network data directory (`~/.frigate/` for mainnet, `~/.frigate/<network>/` otherwise) and to stdout. 
+The default appender does not rotate; in production, configure `logrotate` or run under a service manager that captures stdout (e.g. `systemd-journald`).
+
+### Backup
+
+The tweak index can be rebuilt from Bitcoin Core at any time, but a rebuild from Taproot activation is slow. To minimise downtime, snapshot `db/frigate.duckdb` under the data directory while Frigate is stopped (DuckDB files are portable across operating systems and survive unclean shutdowns). 
+The `config.toml` and any TLS material in the data directory are worth including in the same backup.
+
+### Upgrades
+
+Stop Frigate, install the new release, start. 
+The DuckDB file format and the on-disk index schema are stable across Frigate releases.
+
+### Reorgs
+
+Frigate detects chain reorgs via Bitcoin Core and rescans the affected block range automatically. 
+Connected silent-payments subscribers are renotified for the rescanned range; clients reconnecting after a disconnect should pass `start = lastSeenHeight - 100` on resubscribe to cover any reorg they missed.
+
+## Performance
+
+### CPU Performance
+
+Without GPU acceleration the scanning query is CPU bound, mostly around EC point multiplication.
+[DuckDB parallelizes](https://duckdb.org/docs/stable/guides/performance/how_to_tune_workloads#parallelism-multi-core-processing) the workload based on row groups, with each row group containing 122,880 rows.
+It will by default configure itself to use all the available cores on the server it is running.
+The behaviour can be configured in the Frigate configuration file (see `dbThreads`).
+
+The following results were produced by the included `benchmark.py` script scanning mainnet to block height 914,000.
+Note that a mainnet database indexing from height 800,000 is required to run the benchmark.
+
+M1 MacBook Pro (10 CPUs):
+
+| | Blocks | Start | End | Transactions | Time | Transactions/sec |
+|---|--------|-------|-----|--------------|------|------------------|
+| 2 hours | 12 | 913988 | 914000 | 8,207 | 244ms | 33,608 |
+| 1 day | 144 | 913856 | 914000 | 127,804 | 2s 681ms | 47,675 |
+| 1 week | 1008 | 912992 | 914000 | 751,769 | 3s 600ms | 208,843 |
+| 2 weeks | 2016 | 911984 | 914000 | 1,709,358 | 11s 128ms | 153,602 |
+| 1 month | 4320 | 909680 | 914000 | 4,240,572 | 19s 958ms | 212,470 |
+| 3 months | 12960 | 901040 | 914000 | 13,558,435 | 52s 720ms | 257,179 |
+| 6 months | 25920 | 888080 | 914000 | 26,103,759 | 1m 34s | 274,804 |
+| 1 year | 52560 | 861440 | 914000 | 59,578,156 | 3m 28s | 286,404 |
+| 2 years | 105120 | 808880 | 914000 | 132,994,804 | 7m 47s | 284,342 |
+
+Intel Core Ultra 9 285K (24 CPUs):
+
+| | Blocks | Start | End | Transactions | Time | Transactions/sec |
+|---|--------|-------|-----|--------------|------|------------------|
+| 2 hours | 12 | 913988 | 914000 | 8,207 | 256ms | 32,121 |
+| 1 day | 144 | 913856 | 914000 | 127,804 | 1s 591ms | 80,308 |
+| 1 week | 1008 | 912992 | 914000 | 751,769 | 3s 19ms | 249,026 |
+| 2 weeks | 2016 | 911984 | 914000 | 1,709,358 | 4s 474ms | 382,106 |
+| 1 month | 4320 | 909680 | 914000 | 4,240,572 | 11s 7ms | 385,252 |
+| 3 months | 12960 | 901040 | 914000 | 13,558,435 | 27s 605ms | 491,151 |
+| 6 months | 25920 | 888080 | 914000 | 26,103,759 | 48s 910ms | 533,711 |
+| 1 year | 52560 | 861440 | 914000 | 59,578,156 | 1m 44s | 569,123 |
+| 2 years | 105120 | 808880 | 914000 | 132,994,804 | 3m 50s | 576,610 |
+
+Higher performance on the longer periods is possible by increasing the number of CPUs.
+Multiple clients conducting simultaneous scans slows each scan linearly, since a single scan already saturates all available CPU cores.
+
+### GPU Performance
+
+GPU performance is significantly higher, and as a result is the default compute backend for historical scans.
+
+MacBook M1 Pro (Metal GPU backend):
+
+| | Blocks | Start | End | Transactions | Time | Transactions/sec |
+|---|--------|-------|-----|--------------|------|------------------|
+| 2 hours | 12 | 913988 | 914000 | 8,207 | 32ms | 259,509 |
+| 1 day | 144 | 913856 | 914000 | 127,804 | 240ms | 532,614 |
+| 1 week | 1008 | 912992 | 914000 | 751,769 | 1s 313ms | 572,722 |
+| 2 weeks | 2016 | 911984 | 914000 | 1,709,358 | 3s 91ms | 552,981 |
+| 1 month | 4320 | 909680 | 914000 | 4,240,572 | 7s 458ms | 568,576 |
+| 3 months | 12960 | 901040 | 914000 | 13,558,435 | 23s 288ms | 582,196 |
+| 6 months | 25920 | 888080 | 914000 | 26,103,759 | 44s 575ms | 585,617 |
+| 1 year | 52560 | 861440 | 914000 | 59,578,156 | 1m 41s | 586,138 |
+| 2 years | 105120 | 808880 | 914000 | 132,994,804 | 3m 47s | 584,231 |
+
+NVIDIA RTX 5080 (CUDA backend):
+
+| | Blocks | Start | End | Transactions | Time | Transactions/sec |
+|---|--------|-------|-----|--------------|------|------------------|
+| 2 hours | 12 | 913988 | 914000 | 8,207 | 18ms | 460,614 |
+| 1 day | 144 | 913856 | 914000 | 127,804 | 26ms | 4,880,898 |
+| 1 week | 1008 | 912992 | 914000 | 751,769 | 69ms | 10,906,924 |
+| 2 weeks | 2016 | 911984 | 914000 | 1,709,358 | 146ms | 11,722,414 |
+| 1 month | 4320 | 909680 | 914000 | 4,240,572 | 341ms | 12,447,948 |
+| 3 months | 12960 | 901040 | 914000 | 13,558,435 | 1s 66ms | 12,722,307 |
+| 6 months | 25920 | 888080 | 914000 | 26,103,759 | 1s 613ms | 16,182,843 |
+| 1 year | 52560 | 861440 | 914000 | 59,578,156 | 3s 466ms | 17,188,956 |
+| 2 years | 105120 | 808880 | 914000 | 132,994,804 | 7s 690ms | 17,294,286 |
+
+2x NVIDIA RTX 5090 (CUDA backend):
+
+| | Blocks | Start | End | Transactions | Time | Transactions/sec |
+|---|--------|-------|-----|--------------|------|------------------|
+| 2 hours | 12 | 913988 | 914000 | 8,207 | 21ms | 393,537 |
+| 1 day | 144 | 913856 | 914000 | 127,804 | 53ms | 2,404,792 |
+| 1 week | 1008 | 912992 | 914000 | 751,769 | 92ms | 8,207,103 |
+| 2 weeks | 2016 | 911984 | 914000 | 1,709,358 | 120ms | 14,213,421 |
+| 1 month | 4320 | 909680 | 914000 | 4,240,572 | 158ms | 26,779,011 |
+| 3 months | 12960 | 901040 | 914000 | 13,558,435 | 562ms | 24,146,078 |
+| 6 months | 25920 | 888080 | 914000 | 26,103,759 | 1s 186ms | 22,012,526 |
+| 1 year | 52560 | 861440 | 914000 | 59,578,156 | 2s 96ms | 28,418,826 |
+| 2 years | 105120 | 808880 | 914000 | 132,994,804 | 3s 208ms | 41,455,203 |
+
+This approach is performant enough for a multi-user public instance.
+As EC computation is offloaded to the GPU, CPU overhead is low and normal Electrum server RPC calls can be handled simultaneously without any performance degradation.
+Multiple clients conducting simultaneous scans slows each scan linearly, since a single scan already saturates the available GPUs.
+Using multiple GPUs in the same system is also supported and the workload is scaled across them.
+
+A discrete GPU is not required however.
+Frigate can take advantage of any integrated GPU supported by OpenCL, which in practice includes almost all desktop Intel and AMD chips produced in the last decade.
+This prevents saturation of the CPU, and in the case of weaker CPUs (for example older Intel NUCs) can actually be faster.
+See the section below on enabling the iGPU on Linux.
+
+### GPU Requirements
+
+Frigate supports three GPU backends, selected automatically at runtime in order of preference:
+
+**CUDA (NVIDIA)**
+- NVIDIA Ampere or newer GPU (RTX 30xx, A100, RTX 40xx, H100, RTX 50xx)
+- NVIDIA driver 570.86.15+ (Linux) or 571.14+ (Windows)
+- No CUDA toolkit installation required on the host
+
+**OpenCL (NVIDIA, Intel, AMD)**
+- Any GPU with an OpenCL 1.2+ runtime
+- NVIDIA: OpenCL runtime is included with the driver
+- Intel: requires `intel-opencl-icd` (see [Enabling Intel iGPU on Linux](#enabling-intel-igpu-on-linux))
+- AMD: requires ROCm or AMDGPU-PRO OpenCL runtime
+- On Linux, the `ocl-icd-libopencl1` ICD loader is required
+
+**Metal (Apple)**
+- Apple Silicon (M1 or newer) or AMD GPU with Metal support
+- macOS 12 (Monterey) or newer
+
+### Benchmarking
+
+The `benchmark.py` script in the project root can be used to generate the above tables against a running Frigate server:
+```shell
+python3 benchmark.py
+python3 benchmark.py --markdown
+```
+The `--clients N` option runs N concurrent clients per scan period to test server behaviour under load:
+```shell
+python3 benchmark.py --clients 4
+```
+
+## Enabling Intel iGPU on Linux
+
+Frigate supports GPU-accelerated scanning via OpenCL on Intel integrated GPUs.
+On most Linux distributions, the Intel OpenCL runtime is not installed by default.
+Note that on newer Intel CPUs, the CPU may be stronger than the GPU - but offloading the computation to the GPU is still beneficial.
+
+### Install the Intel OpenCL runtime
+
+On Ubuntu/Debian:
+```shell
+sudo apt install ocl-icd-libopencl1 intel-opencl-icd clinfo
+```
+
+`ocl-icd-libopencl1` is the OpenCL ICD loader that dispatches to vendor runtimes — it may already be installed if another GPU driver (e.g. NVIDIA) is present.
+`intel-opencl-icd` is the Intel GPU compute runtime. Both can coexist with other GPU drivers without affecting them.
+`clinfo` is a diagnostic tool for listing available OpenCL platforms and devices.
+The OpenCL ICD (Installable Client Driver) system allows multiple GPU vendors to coexist.
+
+### Driver compatibility
+
+The Ubuntu-packaged driver may be outdated for newer Intel GPUs.
+If Frigate crashes during startup with a newer Intel GPU, install the latest driver from Intel's PPA:
+```shell
+wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | sudo gpg --yes --dearmor --output /usr/share/keyrings/intel-graphics.gpg
+echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu $(lsb_release -cs) unified" | sudo tee /etc/apt/sources.list.d/intel-gpu.list
+sudo apt update
+sudo apt install intel-opencl-icd
+```
+
+### Verify
+
+Check that the Intel GPU is visible:
+```shell
+clinfo | grep "Device Name"
+```
+
+You should see your Intel GPU listed (e.g. `Intel(R) UHD Graphics [0x7d67]`).
+
+## How It Works
+
+### Motivation
+
+[BIP 352](https://github.com/bitcoin/bips/blob/master/bip-0352.mediawiki) proposes that light clients use compact block filters to scan for UTXOs received to a Silent Payments address.
+This is workable for full nodes but problematic for light clients: a reasonable scan period requires gigabytes of tweaks, filters and blocks to be downloaded, parsed and cached on the client, far exceeding the few megabytes a traditional Electrum wallet uses to restore. 
+It also offers no mempool monitoring, which users rely on to answer the "did you get my transaction?" question.
+
+Frigate addresses both problems by moving scanning to the server, in line with the [Remote Scanner](https://github.com/silent-payments/BIP0352-index-server-specification/blob/main/README.md#remote-scanner-ephemeral) approach in the BIP352 Index Server Specification.
+The scan private key and spend public key are provided to the server, but not stored - they are held in RAM only for the duration of the client session, analogous to how public Electrum servers see ephemeral wallet addresses today.
+
+### The Tweak Index
+
+On startup Frigate connects to Bitcoin Core, downloads blocks from the configured start height (or Taproot activation on mainnet) and builds a tweak index - a single table containing one row per Silent-Payments-eligible transaction:
+
+| Column       | Type         |
+|--------------|--------------|
+| `txid`       | BLOB         |
+| `height`     | INTEGER      |
+| `tweak_key`  | BLOB         |
+| `outputs`    | LIST(BIGINT) |
+
+The `txid` and `tweak_key` values are 32 and 64 byte BLOBs respectively.
+The `outputs` value is a list of 8 byte integers, each representing the first 8 bytes of the x-value of the Taproot output public key.
+
+### In-Database EC Computation
+
+Every silent-payments scan is bounded by EC point multiplication over the tweak index, so Frigate pushes that computation as close to the rows as possible.
+Instead of loading rows into the Frigate server application, scanning is performed by the database itself.
+Frigate uses [DuckDB](https://duckdb.org/why_duckdb.html#fast), a fast OLAP database designed for analytical query workloads, extended with a [custom extension](https://github.com/sparrowwallet/duckdb-ufsecp-extension) wrapping [UltrafastSecp256k1](https://github.com/shrec/UltrafastSecp256k1) - a high-performance secp256k1 library with CPU and GPU backends.
+
+Conceptually, scanning for Silent Payments requires computing the Taproot output key for `k = 0` and comparing it to the list of known output keys for each tweak row:
+```sql
+SELECT txid, tweak_key, height FROM tweak WHERE list_contains(outputs, hash_prefix_to_int(secp256k1_ec_pubkey_combine([SPEND_PUBLIC_KEY, secp256k1_ec_pubkey_create(secp256k1_tagged_sha256('BIP0352/SharedSecret', secp256k1_ec_pubkey_tweak_mul(tweak_key, SCAN_PRIVATE_KEY) || int_to_big_endian(0)))]), 1));
+```
+The client can then download each matching transaction and determine if it does indeed contain outputs it is interested in, including for higher values of `k`.
+
+Frigate performs all of these steps at once using a single scanning function that also includes a further step to scan for change:
+```sql
+SELECT txid, tweak_key, height FROM ufsecp_scan((SELECT txid, height, tweak_key, outputs FROM tweak), SCAN_PRIVATE_KEY, SPEND_PUBLIC_KEY, [CHANGE_TWEAK_KEY]);
+```
+The change tweak is added to the computed P<sub>0</sub> and checked again against the outputs for a match.
+All inputs are provided in little endian format, with public keys in uncompressed little endian x,y format to avoid decompression overhead on each tweak.
+UltrafastSecp256k1 uses batch affine addition and KPlan-optimized scalar multiplication to maximize throughput on CPU, and can leverage CUDA, OpenCL or Metal to offload EC computation to the GPU for significantly higher performance.
+
+## Electrum Protocol
+
+The Electrum protocol is by far the most widely used light client protocol for Bitcoin wallets, and support is now almost a requirement for widespread adoption of any wallet technology proposal.
+It is characterised by resource efficiency for the client in terms of bandwidth, CPU and storage, allowing a good user experience on almost any platform.
+It has however been designed around BIP32 wallets.
+Silent Payments presents an alternative model, where instead of an incrementing derivation path index (and associated gap limit) transactions must be found through scanning the blockchain.
+As such, new methods are necessary.
+Frigate implements the following Electrum JSON-RPC methods, proposed for inclusion in the Electrum protocol:
+
+### server.features
+
+Servers supporting silent payments advertise the capability by including a `silent_payments` field in the `server.features` response:
+
+```json
+{
+  ...,
+  "silent_payments": [0]
+}
+```
+
+Each integer is a fully-specified silent payments protocol version supported by the server.
+BIP352 defines version 0.
+
+### blockchain.silentpayments.subscribe
+
+**Signature**
+```
+blockchain.silentpayments.subscribe(scan_private_key, spend_public_key, start, labels)
+```
+
+- _scan_private_key_: A 64 character string containing the hex of the scan private key.
+- _spend_public_key_: A 66 character string containing the hex of the spend public key.
+- _start_: (Optional) An integer block height to start scanning from, or a string of the form `"FROM-TO"` to specify a closed range of heights. Integer values above 500,000,000 are treated as seconds from the start of the epoch, matching `nLockTime` convention.
+- _labels_: (Optional) An array of positive integers specifying additional silent payment labels to scan for. Change (`m = 0`) is always included regardless. To aid in wallet recovery, this parameter should only be used for specialized applications.
+
+**Result**
+
+A `subscription` JSON object literal containing details of the current subscription:
+- _address_: The silent payment address that has been subscribed to.
+- _labels_: An array of the labels that are subscribed to (must include `0`).
+- _start_height_: The block height from which the subscription scan was started.
+
+If a subscription for the same scan address already exists on the connection, servers **must not** narrow its coverage.
+If the new request's resolved start height is greater than the existing subscription's `start_height`, the server retains the existing wider start and returns it as `start_height` in the response.
+Clients should treat the returned `start_height` as authoritative.
+On every subscribe (including re-subscribe to the same address), the server **must** re-run the historical scan from the returned `start_height` and emit notifications as usual, so that clients which clear their local cache on re-subscribe are repopulated.
+
+**Result Example**
+
+```json
+{
+  "address": "sp1qqgste7k9hx0qftg6qmwlkqtwuy6cycyavzmzj85c6qdfhjdpdjtdgqjuexzk6murw56suy3e0rd2cgqvycxttddwsvgxe2usfpxumr70xc9pkqwv",
+  "labels": [0],
+  "start_height": 882000
+}
+```
+
+### Notifications
+
+Once subscribed, the client will receive notifications as results are returned from the scan with the following signature.
+Servers **must** send the response to `blockchain.silentpayments.subscribe` before any notification for that subscription.
+The `subscription` object in every notification **must** be byte-identical to the one returned in the original `blockchain.silentpayments.subscribe` response with the same `address`, `labels` and `start_height`.
+It identifies which subscription the notification belongs to, not the range that the server happened to scan to produce it.
+All historical (`progress` < `1.0`) results **must** be sent before current (up to date) results.
+A `progress` of `1.0` indicates the scan is up to date as of this notification.
+The first such notification marks the end of the historical scan; subsequent `1.0` notifications represent live updates as new blocks and mempool transactions are scanned.
+Clients should flush any buffered history to local state on every `1.0` notification.
+
+```
+blockchain.silentpayments.subscribe(subscription, progress, history)
+```
+
+**Body**
+
+A dictionary with the following key/value pairs:
+
+1. A `subscription` JSON object literal containing details of the current subscription:
+- _address_: The silent payment address that has been subscribed to.
+- _labels_: An array of the labels that are subscribed to (must include `0`).
+- _start_height_: The block height from which the subscription scan was started.
+
+2. A `progress` key/value pair indicating the progress of a historical scan:
+- _progress_: A floating point value between `0.0` and `1.0`. Will be `1.0` for all current (up to date) results.
+
+3. A `history` array of transactions. Confirmed transactions are listed in order by height, followed by mempool transactions in arbitary order. Each transaction is a dictionary with the following keys:
+- _height_: The integer height of the block the transaction was confirmed in. For mempool transactions, `0` should be used.
+- _tx_hash_: The transaction hash in hexadecimal.
+- _tweak_key_: The tweak key (`input_hash*A`) for the transaction in compressed format.
+
+**Result Example**
+
+```json
+{
+  "subscription": {
+    "address": "sp1qqgste7k9hx0qftg6qmwlkqtwuy6cycyavzmzj85c6qdfhjdpdjtdgqjuexzk6murw56suy3e0rd2cgqvycxttddwsvgxe2usfpxumr70xc9pkqwv",
+    "labels": [0],
+    "start_height": 882000
+  },
+  "progress": 1.0,
+  "history": [
+    {
+      "height": 890004,
+      "tx_hash": "acc3758bd2a26f869fcc67d48ff30b96464d476bca82c1cd6656e7d506816412",
+      "tweak_key": "0314bec14463d6c0181083d607fecfba67bb83f95915f6f247975ec566d5642ee8"
+    },
+    {
+      "height": 905008,
+      "tx_hash": "f3e1bf48975b8d6060a9de8884296abb80be618dc00ae3cb2f6cee3085e09403",
+      "tweak_key": "024ac253c216532e961988e2a8ce266a447c894c781e52ef6cee902361db960004"
+    },
+    {
+      "height": 0,
+      "tx_hash": "f4184fc596403b9d638783cf57adfe4c75c605f6356fbc91338530e9831e9e16",
+      "tweak_key": "03aeea547819c08413974e2ab2b12212e007166bb2058f88b009e082b9b4914a58"
+    }
+  ]
+}
+```
+
+It is recommended that servers implementing this protocol send history results incrementally as the historical scan progresses.
+In addition, a maximum page size of 1000 is suggested when sending historical transactions.
+This will avoid transmission issues with large wallets that have many transactions, while providing the client with regular progress updates.
+Servers should also emit a notification with empty `history` at regular intervals (e.g. every 5 seconds) during a historical scan, to keep the client updated on scanning progress.
+In the case of block reorgs, the server should rescan all existing subscriptions from the reorg-ed block height and send any history (if found) to the client.
+All found mempool transactions should be sent on the initial subscription, but thereafter previously sent mempool transactions should not be resent.
+The server **must not** re-notify a previously sent mempool transaction once it confirms in a block - clients track confirmation state via `blockchain.scripthash.subscribe`.
+The silent payments subscription is purely a discovery channel.
+Once a silent payments subscription has reached `progress = 1.0`, servers **must** deliver silent payments notifications before any scripthash notification arising from the same block or mempool event, since the silent payments subscription is how clients learn which scripthashes to subscribe to.
+
+Clients reconnecting with prior history should pass `start = lastSeenHeight - reorgLimit` to limit the rescan to a recent window.
+A reorg limit of 100 blocks is sufficient.
+
+For every transaction announced via this subscription, clients **must** retrieve it with `blockchain.transaction.get` and subscribe to all owned outputs with `blockchain.scripthash.subscribe`.
+The scripthash subscription is the canonical source of confirmation height, reorg recovery, and spend tracking; the silent payments subscription does not duplicate any of these.
+In other words, the silent payments address subscription is a replacement for the monotonically increasing derivation path index in BIP32 wallets.
+The subscription seeks only to add to the client's knowledge of incoming silent payments transactions.
+The client is responsible for checking the transactions do actually send to addresses it has keys for, and using normal Electrum wallet synchronization techniques to monitor for changes to these addresses.
+The tweak key is provided to allow the client to avoid looking up the scriptPubKeys of spent outputs.
+
+### blockchain.silentpayments.unsubscribe
+
+**Signature**
+```
+blockchain.silentpayments.unsubscribe(scan_private_key, spend_public_key)
+```
+
+- _scan_private_key_: A 64 character string containing the hex of the scan private key.
+- _spend_public_key_: A 66 character string containing the hex of the spend public key.
+
+**Result**
+
+The silent payment address that has been unsubscribed. This should cancel any scans that may be currently running for this address.
+
+**Result Example**
+
+```
+sp1qqgste7k9hx0qftg6qmwlkqtwuy6cycyavzmzj85c6qdfhjdpdjtdgqjuexzk6murw56suy3e0rd2cgqvycxttddwsvgxe2usfpxumr70xc9pkqwv
+```
+
+## Scan Audit Mode
+
+Frigate includes a correctness testing mode to verify the database extension will find all outputs for a given wallet.
+In normal operation, the tweak index stores the hash prefixes derived from the taproot output public keys of each eligible transaction.
+In audit mode, the index instead stores the hash prefix of the expected change output key (P<sub>0</sub> = B<sub>spend</sub> + t<sub>0</sub>·G) computed from the provided wallet keys.
+This treats every silent payments eligible transaction as paying to the provided wallet.
+The audit can then be run with the following query:
+```sql
+SELECT (SELECT COUNT(*) FROM tweak) AS total_rows, COUNT(*) AS matched_rows FROM ufsecp_scan((SELECT txid, height, tweak_key, outputs FROM tweak), from_hex('<scan_private_key_little_endian>'), from_hex('<spend_public_key_little_endian_x_y>'), [from_hex('<change_label_key_little_endian_x_y>')]); 
+```
+
+This mode is activated by setting two environment variables before starting Frigate:
+
+```shell
+export FRIGATE_AUDIT_SCAN_KEY=<scan_private_key_hex>
+export FRIGATE_AUDIT_SPEND_KEY=<spend_public_key_hex>
+```
+
+When both variables are set, a warning is logged on startup confirming audit mode is active.
+The index must be rebuilt from scratch when switching between normal and audit modes.
+
 ## Building
 
 To clone this project, use
@@ -347,8 +715,9 @@ or for those without SSH credentials:
 
 `git clone --recursive https://github.com/sparrowwallet/frigate.git`
 
-In order to build, Frigate requires Java 22 or higher to be installed.
-The release binaries are built with [Eclipse Temurin 22.0.2+9](https://github.com/adoptium/temurin22-binaries/releases/tag/jdk-22.0.2%2B9).
+In order to build, Frigate requires Java 25 or higher to be installed.
+The release binaries are built with [Eclipse Temurin 25.0.2+10](https://github.com/adoptium/temurin25-binaries/releases/tag/jdk-25.0.2%2B10).
+If you are using [SDKMAN](https://sdkman.io/), you can use `sdk env install` to ensure you have the correct version.
 
 Other packages may also be necessary to build depending on the platform. On Debian/Ubuntu systems:
 
@@ -377,4 +746,3 @@ Frigate is licensed under the Apache 2 software licence.
 The Frigate release binaries here are signed using [craigraw's GPG key](https://keybase.io/craigraw):  
 Fingerprint: D4D0D3202FC06849A257B38DE94618334C674B40  
 64-bit: E946 1833 4C67 4B40
-

benchmark.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+
+import argparse
+import concurrent.futures
+import socket
+import json
+import sys
+import time
+
+DEFAULT_HOST = "127.0.0.1"
+DEFAULT_PORT = 57001
+DEFAULT_END_HEIGHT = 914000
+
+SCAN_KEY = "3c8b12d524c72d91dad33573c18f17dddb8f45e8d60c711c49a5a7992e321364"
+SPEND_KEY = "0377dd40dfd0da11369dc6bddf6b3bf4f0474383a8beb6e523dddabc0f966734a6"
+
+# (descriptive label, short label, blocks)
+PERIODS = [
+    ("2 hours",  "2h",  12),
+    ("1 day",    "1d",  144),
+    ("1 week",   "1w",  1008),
+    ("2 weeks",  "2w",  2016),
+    ("1 month",  "1m",  4320),
+    ("3 months", "3m",  12960),
+    ("6 months", "6m",  25920),
+    ("1 year",   "1y",  52560),
+    ("2 years",  "2y",  105120),
+]
+
+# Transaction counts at endHeight=914000 (for display only)
+TRANSACTION_COUNTS = {
+    "2h":  8207,
+    "1d":  127804,
+    "1w":  751769,
+    "2w":  1709358,
+    "1m":  4240572,
+    "3m":  13558435,
+    "6m":  26103759,
+    "1y":  59578156,
+    "2y":  132994804,
+}
+
+
+def scan(host, port, start_range):
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.connect((host, port))
+    sock.settimeout(600)
+
+    sock.sendall(json.dumps({
+        "jsonrpc": "2.0", "method": "server.version",
+        "params": ["benchmark", "1.4"], "id": 1
+    }).encode() + b"\n")
+
+    req = json.dumps({
+        "jsonrpc": "2.0", "method": "blockchain.silentpayments.subscribe",
+        "params": {
+            "scan_private_key": SCAN_KEY,
+            "spend_public_key": SPEND_KEY,
+            "start": start_range,
+        }, "id": 2
+    }).encode() + b"\n"
+
+    t0 = time.monotonic()
+    sock.sendall(req)
+
+    buf = b""
+    while True:
+        data = sock.recv(8192)
+        if not data:
+            break
+        buf += data
+        while b"\n" in buf:
+            line, buf = buf.split(b"\n", 1)
+            msg = json.loads(line)
+            if "params" in msg and msg["params"].get("progress", 0) >= 1.0:
+                elapsed = time.monotonic() - t0
+                sock.close()
+                return elapsed, len(msg["params"].get("history", []))
+            if msg.get("id") == 2 and "error" in msg:
+                sock.close()
+                raise RuntimeError(msg["error"].get("message", str(msg["error"])))
+
+    elapsed = time.monotonic() - t0
+    sock.close()
+    return elapsed, 0
+
+
+def format_time(seconds):
+    ms = round(seconds * 1000)
+    if ms < 1000:
+        return f"{ms}ms"
+    s = ms // 1000
+    remainder_ms = ms % 1000
+    if s < 60:
+        return f"{s}s {remainder_ms}ms"
+    m = s // 60
+    remainder_s = s % 60
+    return f"{m}m {remainder_s}s"
+
+
+def format_number(n):
+    return f"{n:,}"
+
+
+def run_benchmarks(host, port, end_height, markdown, clients, max_periods=0):
+    period_list = PERIODS[:max_periods] if max_periods > 0 else PERIODS
+    periods = []
+    for desc, short, blocks in period_list:
+        start = end_height - blocks
+        txns = TRANSACTION_COUNTS.get(short) if end_height == DEFAULT_END_HEIGHT else None
+        periods.append((desc, short, blocks, start, f"{start}-{end_height}", txns))
+
+    # Warmup scan (first scan on a new connection has overhead from
+    # precompute table loading in the DuckDB read pool)
+    print("Warming up...", end="", flush=True)
+    scan(host, port, periods[0][4])
+    print(" done.\n")
+
+    results = []
+    for desc, short, blocks, start, height_range, txns in periods:
+        sys.stdout.write(f"  Scanning {short}...")
+        sys.stdout.flush()
+        if clients == 1:
+            elapsed, count = scan(host, port, height_range)
+        else:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=clients) as pool:
+                futures = [pool.submit(scan, host, port, height_range) for _ in range(clients)]
+                thread_results = [f.result() for f in futures]
+            elapsed = max(e for e, c in thread_results)
+        tps = round(clients * txns / elapsed) if txns and elapsed > 0 else None
+        results.append((desc, blocks, start, end_height, txns, elapsed, tps))
+        sys.stdout.write(f" {format_time(elapsed)}\n")
+        sys.stdout.flush()
+
+    print()
+    print_results(results, markdown, clients)
+
+
+def print_results(results, markdown, clients=1):
+    show_txns = any(r[4] is not None for r in results)
+    tps_label = f"Transactions/sec ({clients} clients)" if clients > 1 else "Transactions/sec"
+
+    if markdown:
+        if show_txns:
+            print(f"| | Blocks | Start | End | Transactions | Time | {tps_label} |")
+            print("|---|--------|-------|-----|--------------|------|------------------|")
+            for desc, blocks, start, end, txns, elapsed, tps in results:
+                print(f"| {desc} | {blocks} | {start} | {end} | {format_number(txns)} | {format_time(elapsed)} | {format_number(tps)} |")
+        else:
+            print("| | Blocks | Start | End | Time |")
+            print("|---|--------|-------|-----|------|")
+            for desc, blocks, start, end, txns, elapsed, tps in results:
+                print(f"| {desc} | {blocks} | {start} | {end} | {format_time(elapsed)} |")
+    else:
+        if show_txns:
+            h = ("", "Blocks", "Start", "End", "Transactions", "Time", tps_label)
+            rows = [(desc, str(blocks), str(start), str(end), format_number(txns), format_time(elapsed), format_number(tps)) for desc, blocks, start, end, txns, elapsed, tps in results]
+        else:
+            h = ("", "Blocks", "Start", "End", "Time")
+            rows = [(desc, str(blocks), str(start), str(end), format_time(elapsed)) for desc, blocks, start, end, txns, elapsed, tps in results]
+
+        widths = [max(len(h[i]), max(len(r[i]) for r in rows)) for i in range(len(h))]
+
+        def fmt_row(vals):
+            parts = []
+            for i, v in enumerate(vals):
+                if i == 0:
+                    parts.append(f"{v:<{widths[i]}}")
+                else:
+                    parts.append(f"{v:>{widths[i]}}")
+            print("  " + "  ".join(parts))
+
+        fmt_row(h)
+        fmt_row(tuple("─" * w for w in widths))
+        for r in rows:
+            fmt_row(r)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Benchmark Frigate Silent Payments scanning performance.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="Examples:\n  python3 benchmark.py\n  python3 benchmark.py --end-height 920000 --markdown\n  python3 benchmark.py --host 192.168.1.10 --port 57001\n  python3 benchmark.py --clients 4",
+    )
+    parser.add_argument("--host", default=DEFAULT_HOST, help=f"server host (default: {DEFAULT_HOST})")
+    parser.add_argument("--port", type=int, default=DEFAULT_PORT, help=f"server port (default: {DEFAULT_PORT})")
+    parser.add_argument("--end-height", type=int, default=DEFAULT_END_HEIGHT, help=f"end block height (default: {DEFAULT_END_HEIGHT})")
+    parser.add_argument("--markdown", action="store_true", help="output as markdown table")
+    parser.add_argument("--clients", type=int, default=1, help="number of concurrent clients per scan period (default: 1)")
+    parser.add_argument("--periods", type=int, default=0, help="number of periods to run (default: all)")
+    args = parser.parse_args()
+
+    try:
+        run_benchmarks(args.host, args.port, args.end_height, args.markdown, args.clients, args.periods)
+    except ConnectionRefusedError:
+        print(f"Error: could not connect to {args.host}:{args.port}", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

build.gradle

@@ -1,25 +1,28 @@
 plugins {
     id 'application'
-    id 'org.gradlex.extra-java-module-info' version '1.13'
-    id 'org.beryx.jlink' version '3.1.2'
+    id 'org.gradlex.extra-java-module-info' version '1.13.1'
+    id 'org.beryx.jlink' version '3.2.1'
 }
 
-org.gradle.internal.os.OperatingSystem os = org.gradle.internal.os.OperatingSystem.current()
-String releaseArch = System.getProperty('os.arch') == 'aarch64' ? 'aarch64' : 'x86_64'
+def os = org.gradle.internal.os.OperatingSystem.current()
+def releaseArch = System.getProperty('os.arch') == 'aarch64' ? 'aarch64' : 'x86_64'
 
 group = 'com.sparrowwallet.frigate'
-version = '1.0.0'
+version = '1.5.3'
 
 repositories {
     mavenCentral()
 }
 
+tasks.withType(AbstractArchiveTask).configureEach {
+    useFileSystemPermissions()
+}
+
 dependencies {
     implementation(project(':drongo'))
-    implementation('org.duckdb:duckdb_jdbc:1.3.2.0')
-    implementation("com.zaxxer:HikariCP:7.0.2")
-    implementation('com.google.guava:guava:33.0.0-jre')
-    implementation('com.google.code.gson:gson:2.9.1')
+    implementation('org.duckdb:duckdb_jdbc:1.4.4.0')
+    implementation('com.google.guava:guava:33.5.0-jre')
+    implementation('com.google.code.gson:gson:2.13.2')
     implementation('com.github.arteam:simple-json-rpc-core:1.3')
     implementation('com.github.arteam:simple-json-rpc-client:1.3') {
         exclude group: 'com.github.arteam', module: 'simple-json-rpc-core'
@@ -27,11 +30,17 @@ dependencies {
     implementation('com.github.arteam:simple-json-rpc-server:1.3') {
         exclude group: 'org.slf4j'
     }
-    implementation('com.fasterxml.jackson.core:jackson-databind:2.17.2')
-    implementation('org.jcommander:jcommander:2.0')
-    implementation ('org.slf4j:slf4j-api:2.0.12')
-    testImplementation platform('org.junit:junit-bom:5.10.0')
+    implementation('com.fasterxml.jackson.core:jackson-databind:2.21.1')
+    implementation('com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.21.1')
+    implementation('org.jcommander:jcommander:3.0')
+    implementation('org.zeromq:jeromq:0.6.0')
+    implementation ('org.slf4j:slf4j-api:2.0.17')
+    implementation ('ch.qos.logback:logback-classic:1.5.32') {
+        exclude group: 'org.slf4j'
+    }
+    testImplementation platform('org.junit:junit-bom:5.14.1')
     testImplementation 'org.junit.jupiter:junit-jupiter'
+    testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
 }
 
 test {
@@ -41,6 +50,20 @@ test {
 application {
     mainModule = 'com.sparrowwallet.frigate'
     mainClass = 'com.sparrowwallet.frigate.Frigate'
+
+    applicationDefaultJvmArgs = ["-XX:+HeapDumpOnOutOfMemoryError",
+                                 "--enable-native-access=duckdb.jdbc,com.sparrowwallet.drongo"]
+    if(os.isMacOsX()) {
+        applicationDefaultJvmArgs += ["-Dapple.awt.UIElement=true"]
+    }
+}
+
+tasks.register('runCli', JavaExec) {
+    mainModule = 'com.sparrowwallet.frigate'
+    mainClass = 'com.sparrowwallet.frigate.cli.FrigateCli'
+    classpath = sourceSets.main.runtimeClasspath
+    jvmArgs = ["-XX:+HeapDumpOnOutOfMemoryError",
+               "--enable-native-access=duckdb.jdbc,com.sparrowwallet.drongo"]
 }
 
 jlink {
@@ -49,13 +72,26 @@ jlink {
         requires 'java.logging'
     }
 
-    options = ['--strip-native-commands', '--strip-java-debug-attributes', '--compress', 'zip-6', '--no-header-files', '--no-man-pages', '--ignore-signing-information', '--exclude-files', '**.png', '--exclude-resources', 'glob:/com.sparrowwallet.merged.module/META-INF/*']
+    def nativePlatform = os.isMacOsX() ? 'macos' : os.isWindows() ? 'windows' : 'linux'
+    def nativeArch = releaseArch == 'aarch64' ? 'arm64' : 'amd64'
+    def excludeGlobs = (['linux', 'macos', 'windows'].collectMany { osName ->
+        ['amd64', 'arm64'].findResults { arch ->
+            (osName == nativePlatform && arch == nativeArch) ? null : "glob:/com.sparrowwallet.frigate/native/${osName}/${arch}/**"
+        }
+    } + ['glob:/com.sparrowwallet.merged.module/META-INF/*']).join(',')
+    def jlinkOptions = ['--strip-native-commands', '--strip-java-debug-attributes', '--compress', 'zip-6', '--no-header-files', '--no-man-pages', '--ignore-signing-information', '--exclude-files', '**.png', '--exclude-resources', excludeGlobs]
+    if(os.isMacOsX()) {
+        jlinkOptions += ['--add-modules', 'java.desktop']
+    }
+    options = jlinkOptions
     launcher {
         name = 'frigate'
+        jvmArgs = ["--enable-native-access=duckdb.jdbc,com.sparrowwallet.drongo"]
     }
     secondaryLauncher {
         name = 'frigate-cli'
         mainClass = 'com.sparrowwallet.frigate.cli.FrigateCli'
+        jvmArgs = ["--enable-native-access=duckdb.jdbc,com.sparrowwallet.drongo"]
     }
     jpackage {
         imageName = os.isMacOsX() ? 'Frigate' : 'frigate'
@@ -65,7 +101,7 @@ jlink {
         if(os.isWindows()) {
             imageOptions = ['--win-console', '--resource-dir', 'src/main/deploy/package/windows/']
         } else if(os.isMacOsX()) {
-            imageOptions = ['--icon', 'src/main/deploy/package/macos/frigate.icns', '--resource-dir', 'src/main/deploy/package/macos/']
+            imageOptions = ['--icon', 'src/main/deploy/package/macos/frigate.icns', '--resource-dir', 'src/main/deploy/package/macos/', '--java-options', '-Dapple.awt.UIElement=true']
         } else if(os.isLinux()) {
             imageOptions = ['--resource-dir', 'src/main/deploy/package/linux/', '--icon', 'src/main/deploy/package/linux/frigate.png']
             installerOptions = ['--linux-rpm-license-type', 'ASL 2.0', '--linux-deb-maintainer', 'mail@sparrowwallet.com']
@@ -116,7 +152,7 @@ tasks.register('packageTarDistribution', Tar) {
 }
 
 extraJavaModuleInfo {
-    module('org.duckdb:duckdb_jdbc', 'org.duckdb.duckdb_jdbc') {
+    module('org.duckdb:duckdb_jdbc', 'duckdb.jdbc') {
         exports('org.duckdb')
         requires('java.sql')
     }
@@ -149,4 +185,7 @@ extraJavaModuleInfo {
     module('org.jcommander:jcommander', 'org.jcommander') {
         exports('com.beust.jcommander')
     }
+    module('eu.neilalexander:jnacl', 'jnacl') {
+        exports('com.neilalexander.jnacl.crypto')
+    }
 }

drongo

@@ -1 +1 @@
-Subproject commit 0b3b1a5c3f45e1c81cdc15ea565c194ef615f814
+Subproject commit dad9fe2fccda624566e44324d0da42a3423f8e6e

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 #
-# Copyright © 2015-2021 the original authors.
+# Copyright © 2015 the original authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -114,7 +114,6 @@ case "$( uname )" in                #(
   NONSTOP* )        nonstop=true ;;
 esac
 
-CLASSPATH="\\\"\\\""
 
 
 # Determine the Java command to use to start the JVM.
@@ -172,7 +171,6 @@ fi
 # For Cygwin or MSYS, switch paths to Windows format before running java
 if "$cygwin" || "$msys" ; then
     APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
-    CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
 
     JAVACMD=$( cygpath --unix "$JAVACMD" )
 
@@ -212,7 +210,6 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \
-        -classpath "$CLASSPATH" \
         -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
         "$@"
 

gradlew.bat

@@ -70,11 +70,10 @@ goto fail
 :execute
 @rem Setup the command line
 
-set CLASSPATH=
 
 
 @rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %*
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %*
 
 :end
 @rem End local scope for the variables with windows NT shell

src/main/java/com/sparrowwallet/frigate/Frigate.java

@@ -8,17 +8,28 @@ import com.sparrowwallet.frigate.electrum.ElectrumServerRunnable;
 import com.sparrowwallet.frigate.bitcoind.BitcoindClient;
 import com.sparrowwallet.frigate.index.Index;
 import com.sparrowwallet.frigate.index.IndexQuerier;
+import com.sparrowwallet.drongo.OsType;
 import com.sparrowwallet.frigate.io.Config;
+import com.sparrowwallet.frigate.io.Server;
+import com.sparrowwallet.frigate.io.SslUtil;
 import com.sparrowwallet.frigate.io.Storage;
+import javax.net.ssl.SSLContext;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import com.github.arteam.simplejsonrpc.client.exception.JsonRpcException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.IOException;
+import java.net.BindException;
+import java.net.ConnectException;
+import java.sql.SQLException;
 import java.util.Locale;
 
 public class Frigate {
     public static final String SERVER_NAME = "Frigate";
-    public static final String SERVER_VERSION = "1.0.0";
+    public static final String SERVER_VERSION = "1.5.3";
     public static final String APP_HOME_PROPERTY = "frigate.home";
     public static final String NETWORK_ENV_PROPERTY = "FRIGATE_NETWORK";
     private static final int MAINNET_TAPROOT_ACTIVATION_HEIGHT = 709632;
@@ -28,35 +39,43 @@ public class Frigate {
 
     private Index blocksIndex;
     private Index mempoolIndex;
+    private IndexQuerier indexQuerier;
     private BitcoindClient bitcoindClient;
     private ElectrumServerRunnable electrumServer;
 
     private boolean running;
 
+    private static Object trayManager;
+
     public void start() {
+        getLogger().info("Starting " + SERVER_NAME + " v" + SERVER_VERSION + " on " + Network.get().getName());
+        getLogger().info("Using home directory " + Storage.getFrigateHome().getAbsolutePath());
+
         Runtime.getRuntime().addShutdownHook(new Thread(this::stop));
 
-        Integer startHeight = Config.get().getIndexStartHeight();
+        Config config = Config.get();
+
+        Integer startHeight = config.getIndex().getStartHeight();
         if(startHeight == null) {
             startHeight = Network.get() == Network.MAINNET ? MAINNET_TAPROOT_ACTIVATION_HEIGHT : TESTNET_TAPROOT_ACTIVATION_HEIGHT;
-            Config.get().setIndexStartHeight(startHeight);
         }
 
-        blocksIndex = new Index(startHeight, false);
-        mempoolIndex = new Index(0, true);
+        int batchSize = config.getScan().getBatchSize();
 
-        Boolean startIndexing = Config.get().isStartIndexing();
-        if(startIndexing == null) {
-            startIndexing = true;
-            Config.get().setStartIndexing(startIndexing);
-        }
+        blocksIndex = new Index(startHeight, false, batchSize);
+        mempoolIndex = new Index(0, true, batchSize);
 
-        if(startIndexing) {
+        if(config.getCore().shouldConnect()) {
             bitcoindClient = new BitcoindClient(blocksIndex, mempoolIndex);
             bitcoindClient.initialize();
         }
 
-        electrumServer = new ElectrumServerRunnable(bitcoindClient, new IndexQuerier(blocksIndex, mempoolIndex));
+        Config.ServerConfig serverConfig = config.getServer();
+        SSLContext sslContext = serverConfig.isSslEnabled() ? SslUtil.getServerSSLContext(serverConfig.getSslCertFile(), serverConfig.getSslKeyFile()) : null;
+        InetSocketAddress tcpBind = toBindAddress(serverConfig.getTcpServer());
+        InetSocketAddress sslBind = toBindAddress(serverConfig.getSslServer());
+        indexQuerier = new IndexQuerier(blocksIndex, mempoolIndex);
+        electrumServer = new ElectrumServerRunnable(bitcoindClient, indexQuerier, tcpBind, sslBind, sslContext);
         Thread electrumServerThread = new Thread(electrumServer, "Frigate Electrum Server");
         electrumServerThread.setDaemon(false);
         electrumServerThread.start();
@@ -69,6 +88,8 @@ public class Frigate {
     }
 
     public void stop() {
+        getLogger().info(SERVER_NAME + " shutting down...");
+
         if(blocksIndex != null) {
             blocksIndex.close();
         }
@@ -81,6 +102,9 @@ public class Frigate {
         if(electrumServer != null) {
             electrumServer.stop();
         }
+        if(indexQuerier != null) {
+            indexQuerier.close();
+        }
 
         running = false;
     }
@@ -89,10 +113,30 @@ public class Frigate {
         return EVENT_BUS;
     }
 
+    private static InetSocketAddress toBindAddress(Server server) {
+        if(server == null) {
+            return null;
+        }
+
+        try {
+            return server.getInetSocketAddress();
+        } catch(UnknownHostException e) {
+            throw new ConfigurationException("Cannot resolve listener bind host '" + server.getHost() + "': " + e.getMessage(), e);
+        }
+    }
+
     private static Logger getLogger() {
         return LoggerFactory.getLogger(Frigate.class);
     }
 
+    private static void initTray() {
+        if(com.sparrowwallet.frigate.control.TrayManager.isSupported()) {
+            com.sparrowwallet.frigate.control.TrayManager mgr = new com.sparrowwallet.frigate.control.TrayManager();
+            EVENT_BUS.register(mgr);
+            trayManager = mgr;
+        }
+    }
+
     public static void main(String[] argv) {
         Args args = new Args();
         JCommander jCommander = JCommander.newBuilder().addObject(args).programName(SERVER_NAME.toLowerCase(Locale.ROOT)).acceptUnknownOptions(true).build();
@@ -148,7 +192,73 @@ public class Frigate {
             getLogger().info("Using " + Network.get() + " configuration");
         }
 
-        Frigate frigate = new Frigate();
-        frigate.start();
+        try {
+            if(OsType.getCurrent() == OsType.MACOS) {
+                initTray();
+            }
+
+            Frigate frigate = new Frigate();
+            frigate.start();
+        } catch(Exception e) {
+            String message = getOperationalErrorMessage(e);
+            if(message != null) {
+                getLogger().error(message);
+            } else {
+                getLogger().error("Fatal error", e);
+            }
+            System.exit(1);
+        }
+    }
+
+    private static String getOperationalErrorMessage(Exception e) {
+        String configExceptionMessage = null;
+        Throwable current = e;
+        while(current != null) {
+            if(current instanceof ConnectException) {
+                String server = Config.get().getCore().getServer();
+                if(server == null) server = "http://127.0.0.1:8332";
+                return "Cannot connect to Bitcoin Core at " + server + ". Ensure Bitcoin Core is running and the server URL in config.toml is correct, or set connect = false under [core].";
+            }
+            if(current instanceof BindException) {
+                Config.ServerConfig sc = Config.get().getServer();
+                Server tcpServer = sc.getTcpServer();
+                Server sslServer = sc.getSslServer();
+                String msg = current.getMessage();
+                String which;
+                if(tcpServer != null && sslServer != null) {
+                    int sslPort = sslServer.getHostAndPort().getPort();
+                    which = msg != null && msg.contains(Integer.toString(sslPort)) ? "SSL at " + sslServer.getUrl() : "TCP at " + tcpServer.getUrl();
+                } else if(tcpServer != null) {
+                    which = "TCP at " + tcpServer.getUrl();
+                } else {
+                    which = "SSL at " + sslServer.getUrl();
+                }
+                return "Configured Electrum server listener (" + which + ") is already in use. Another Frigate instance may be running, or change the listener under [server] in config.toml.";
+            }
+            if(current instanceof IOException ioe && ioe.getMessage() != null) {
+                String msg = ioe.getMessage();
+                if(msg.contains("Cannot find Bitcoin Core cookie file")) {
+                    return msg + ". Ensure Bitcoin Core is running, or set connect = false under [core] in config.toml.";
+                }
+                if(msg.contains("authentication failed")) {
+                    return "Bitcoin Core authentication failed. Check authType, dataDir, and auth under [core] in config.toml, or set connect = false.";
+                }
+            }
+            if(current instanceof SQLException sql && sql.getMessage() != null) {
+                if(sql.getMessage().contains("Could not set lock")) {
+                    return "Database is locked by another process. Ensure no other Frigate instance is using the same database.";
+                }
+            }
+            if(current instanceof JsonRpcException rpc && rpc.getMessage() != null) {
+                if(rpc.getMessage().contains("-txindex")) {
+                    return "Bitcoin Core requires txindex=1 in bitcoin.conf. Restart Bitcoin Core after adding it.";
+                }
+            }
+            if(current instanceof ConfigurationException && configExceptionMessage == null) {
+                configExceptionMessage = current.getMessage();
+            }
+            current = current.getCause();
+        }
+        return configExceptionMessage;
     }
 }

src/main/java/com/sparrowwallet/frigate/bitcoind/BitcoindClient.java

@@ -1,6 +1,7 @@
 package com.sparrowwallet.frigate.bitcoind;
 
 import com.github.arteam.simplejsonrpc.client.JsonRpcClient;
+import com.github.arteam.simplejsonrpc.client.exception.JsonRpcBatchException;
 import com.github.arteam.simplejsonrpc.client.exception.JsonRpcException;
 import com.sparrowwallet.drongo.Network;
 import com.sparrowwallet.drongo.OsType;
@@ -16,9 +17,20 @@ import com.sparrowwallet.frigate.io.RecentBlocksMap;
 import com.sparrowwallet.frigate.io.Server;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.zeromq.SocketType;
+import org.zeromq.ZContext;
+import org.zeromq.ZMQ;
 
 import java.io.File;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
@@ -26,17 +38,27 @@ import java.util.concurrent.locks.ReentrantLock;
 public class BitcoindClient {
     private static final Logger log = LoggerFactory.getLogger(BitcoindClient.class);
 
-    public static final int DEFAULT_SCRIPT_PUB_KEY_CACHE_SIZE = 10000000;
     private static final int MAX_REORG_DEPTH = 10;
+    private static final int MIN_GETBLOCK_VERBOSE_VERSION = 250000;
+    public static final int MIN_SUBMIT_PACKAGE_VERSION = 280000;
 
+    private static final int FLUSH_BATCH_SIZE = 50;
+    private static final long FLUSH_DEBOUNCE_MS = 100;
+    private static final long ZMQ_MEMPOOL_DIFF_INTERVAL_MS = 30_000;
+    private static final long POLL_MEMPOOL_DIFF_INTERVAL_MS = 5_000;
+    private static final long ZMQ_STALENESS_THRESHOLD_MS = 60_000;
+
+    private final BitcoindTransport bitcoindTransport;
     private final JsonRpcClient jsonRpcClient;
+    private final AtomicLong rpcIdCounter = new AtomicLong();
     private final Timer timer = new Timer(true);
     private final Index blocksIndex;
     private final Index mempoolIndex;
 
     private NetworkInfo networkInfo;
     private String lastBlock;
-    private ElectrumBlockHeader tip;
+    private volatile ElectrumBlockHeader tip;
+    private volatile boolean useGetBlockVerbose;
 
     private Exception lastPollException;
 
@@ -44,67 +66,79 @@ public class BitcoindClient {
     private final Condition syncingCondition = syncingLock.newCondition();
     private boolean syncing;
 
-    private boolean stopped;
+    private volatile boolean stopped;
+
+    private volatile ZContext zmqContext;
+    private volatile Thread zmqSubscriberThread;
+    private volatile Thread zmqConsumerThread;
+    private volatile long lastZmqMessageMs;
+    private final BlockingQueue<MempoolSeqEvent> zmqQueue = new LinkedBlockingQueue<>(50_000);
+    private final AtomicBoolean pollPending = new AtomicBoolean();
+    private long lastMempoolDiffMs;
 
     private final Map<HashIndex, byte[]> scriptPubKeyCache;
-    private final Set<Sha256Hash> mempoolTxIds = new HashSet<>();
+    private final Set<Sha256Hash> mempoolTxIds = ConcurrentHashMap.newKeySet();
     private final RecentBlocksMap recentBlocksMap = new RecentBlocksMap(MAX_REORG_DEPTH);
 
     public BitcoindClient(Index blocksIndex, Index mempoolIndex) {
-        BitcoindTransport bitcoindTransport;
-
-        Config config = Config.get();
-        Server coreServer = config.getCoreServer();
-        if(coreServer == null) {
-            coreServer = new Server("http://127.0.0.1:" + Network.get().getDefaultPort());
-            Config.get().setCoreServer(coreServer);
-        }
-
-        CoreAuthType coreAuthType = config.getCoreAuthType();
-        if(coreAuthType == null) {
-            coreAuthType = CoreAuthType.COOKIE;
-            Config.get().setCoreAuthType(coreAuthType);
-        }
-
-        File coreDataDir = config.getCoreDataDir();
-        if(coreDataDir == null) {
-            coreDataDir = getDefaultCoreDataDir();
-            Config.get().setCoreDataDir(coreDataDir);
-        }
-
-        String coreAuth = config.getCoreAuth();
-        if(coreAuth == null) {
-            coreAuth = "user:password";
-            Config.get().setCoreAuth(coreAuth);
-        }
-
-        if(coreAuthType == CoreAuthType.COOKIE || coreAuth.length() < 2) {
-            bitcoindTransport = new BitcoindTransport(coreServer, coreDataDir);
-        } else {
-            bitcoindTransport = new BitcoindTransport(coreServer, coreAuth);
-        }
+        this(blocksIndex, mempoolIndex, buildDefaultTransport());
+    }
 
+    BitcoindClient(Index blocksIndex, Index mempoolIndex, BitcoindTransport bitcoindTransport) {
+        this.bitcoindTransport = bitcoindTransport;
         this.jsonRpcClient = new JsonRpcClient(bitcoindTransport);
         this.blocksIndex = blocksIndex;
         this.mempoolIndex = mempoolIndex;
 
-        Integer cacheSize = Config.get().getScriptPubKeyCacheSize();
-        if(cacheSize == null) {
-            cacheSize = DEFAULT_SCRIPT_PUB_KEY_CACHE_SIZE;
-            Config.get().setScriptPubKeyCacheSize(cacheSize);
+        int cacheSize = Config.get().getIndex().getCacheSizeEntries();
+        this.scriptPubKeyCache = Collections.synchronizedMap(lruCache(cacheSize));
+    }
+
+    private static BitcoindTransport buildDefaultTransport() {
+        Config.CoreConfig coreConfig = Config.get().getCore();
+
+        Server coreServer = coreConfig.getServerObj();
+        if(coreServer == null) {
+            coreServer = new Server("http://127.0.0.1:" + Network.get().getDefaultPort());
         }
-        this.scriptPubKeyCache = lruCache(cacheSize);
+
+        CoreAuthType coreAuthType = coreConfig.getAuthTypeEnum();
+        if(coreAuthType == null) {
+            coreAuthType = CoreAuthType.COOKIE;
+        }
+
+        File coreDataDir = coreConfig.getDataDirFile();
+        if(coreDataDir == null) {
+            coreDataDir = getDefaultCoreDataDir();
+        }
+
+        String coreAuth = coreConfig.getAuth();
+        if(coreAuth == null) {
+            coreAuth = "user:password";
+        }
+
+        if(coreAuthType == CoreAuthType.COOKIE || coreAuth.length() < 2) {
+            return new BitcoindTransport(coreServer, coreDataDir);
+        }
+
+        return new BitcoindTransport(coreServer, coreAuth);
     }
 
     public void initialize() {
         networkInfo = getBitcoindService().getNetworkInfo();
 
         BlockchainInfo blockchainInfo = getBitcoindService().getBlockchainInfo();
+        useGetBlockVerbose = networkInfo.version() >= MIN_GETBLOCK_VERBOSE_VERSION && !blockchainInfo.pruned();
+        if(useGetBlockVerbose) {
+            log.debug("Using getblock verbosity=3 (Bitcoin Core {})", networkInfo.version());
+        }
+
         VerboseBlockHeader blockHeader = getBitcoindService().getBlockHeader(blockchainInfo.bestblockhash());
         tip = blockHeader.getBlockHeader();
         timer.schedule(new PollTask(), 5000, 5000);
 
         if(blockchainInfo.initialblockdownload() && networkInfo.networkactive()) {
+            log.info("Waiting for sync to complete...");
             syncingLock.lock();
             try {
                 syncing = true;
@@ -128,104 +162,467 @@ public class BitcoindClient {
         }
 
         lastBlock = blockchainInfo.bestblockhash();
-        log.info("Initializing indexes...");
+        Frigate.getEventBus().post(tip);
+
+        blocksIndex.repairOrphanTweaks();
+
+        int markerHeight = blocksIndex.getLastBlockIndexed();
+        byte[] storedHash = blocksIndex.getLastBlockHash();
+        if(markerHeight > 0 && storedHash != null) {
+            String currentHashHex = getBitcoindService().getBlockHash(markerHeight);
+            byte[] currentHash = Sha256Hash.wrap(currentHashHex).getBytes();
+            if(!Arrays.equals(storedHash, currentHash)) {
+                int rollbackTo = Math.max(0, markerHeight - MAX_REORG_DEPTH);
+                log.info("Stored block hash at height {} does not match bitcoind - rolling back to height {} and re-indexing", markerHeight, rollbackTo);
+                blocksIndex.removeFromIndex(rollbackTo + 1);
+            }
+        }
+
+        int startHeight = blocksIndex.getLastBlockIndexed() + 1;
+        int endHeight = tip.height();
+        int blocksToIndex = endHeight - startHeight + 1;
+        if(blocksToIndex > 0) {
+            log.info("Indexing {} blocks ({} to {})...", blocksToIndex, startHeight, endHeight);
+        } else {
+            log.info("Block index is up to date");
+        }
+        long startTime = System.currentTimeMillis();
         updateBlocksIndex();
+        if(blocksToIndex > 0) {
+            long elapsedMs = System.currentTimeMillis() - startTime;
+            double blocksPerSec = blocksToIndex / (elapsedMs / 1000.0);
+            log.info("Indexed {} blocks in {}.{}s ({} blocks/sec)", blocksToIndex, elapsedMs / 1000, String.format(Locale.ROOT, "%03d", elapsedMs % 1000), String.format(Locale.ROOT, "%.1f", blocksPerSec));
+        }
+        blocksIndex.setSteadyState(true);
         updateMempoolIndex();
-    }
+        lastMempoolDiffMs = System.currentTimeMillis();
+        Frigate.getEventBus().post(tip);
 
-    private synchronized void updateBlocksIndex() {
-        BitcoindClientService bitcoindService = getBitcoindService();
-        HexFormat hexFormat = HexFormat.of();
-
-        for(int i = blocksIndex.getLastBlockIndexed() + 1; i <= tip.height(); i++) {
-            String blockHash = getBitcoindService().getBlockHash(i);
-            if(i > tip.height() - MAX_REORG_DEPTH) {
-                recentBlocksMap.put(i, blockHash);
-            }
-            String blockHex = (String)bitcoindService.getBlock(blockHash, 0);
-            Block block = new Block(hexFormat.parseHex(blockHex));
-
-            Map<BlockTransaction, byte[]> eligibleTransactions = new LinkedHashMap<>();
-            Map<HashIndex, Script> spentScriptPubKeys = new HashMap<>();
-            for(Transaction tx : block.getTransactions()) {
-                for(int outputIndex = 0; outputIndex < tx.getOutputs().size(); outputIndex++) {
-                    byte[] scriptPubKeyBytes = tx.getOutputs().get(outputIndex).getScriptBytes();
-                    addtoScriptPubKeyCache(tx.getTxId(), outputIndex, scriptPubKeyBytes);
-                }
-
-                if(!tx.isCoinBase() && containsTaprootOutput(tx)) {
-                    for(TransactionInput txInput : tx.getInputs()) {
-                        HashIndex hashIndex = new HashIndex(txInput.getOutpoint().getHash(), txInput.getOutpoint().getIndex());
-                        spentScriptPubKeys.put(hashIndex, getScriptPubKey(bitcoindService, hexFormat, hashIndex));
-                    }
-
-                    byte[] tweak = SilentPaymentUtils.getTweak(tx, spentScriptPubKeys);
-                    if(tweak != null) {
-                        BlockTransaction blkTx = new BlockTransaction(tx.getTxId(), i, block.getBlockHeader().getTimeAsDate(), 0L, tx, block.getHash());
-                        eligibleTransactions.put(blkTx, tweak);
-                    }
-                }
-            }
-
-            if(!eligibleTransactions.isEmpty()) {
-                blocksIndex.addToIndex(eligibleTransactions);
+        String zmqEndpoint = Config.get().getCore().getZmqSequenceEndpoint();
+        boolean autoDiscoveryAttempted = false;
+        boolean zmqUnsupported = false;
+        if((zmqEndpoint == null || zmqEndpoint.isBlank()) && isLoopbackBitcoind()) {
+            autoDiscoveryAttempted = true;
+            ZmqDiscovery discovery = discoverZmqSequenceEndpoint();
+            zmqEndpoint = discovery.endpoint();
+            zmqUnsupported = discovery.unsupported();
+        }
+        if(zmqEndpoint != null && !zmqEndpoint.isBlank()) {
+            startZmqSequenceSubscriber(zmqEndpoint);
+        } else {
+            long pollSeconds = POLL_MEMPOOL_DIFF_INTERVAL_MS / 1000;
+            String fix = zmqUnsupported ? "This Bitcoin Core binary was built without ZMQ support — install a release build, or rebuild with ZeroMQ (-DWITH_ZMQ=ON)"
+                    : autoDiscoveryAttempted ? "Add -zmqpubsequence=tcp://127.0.0.1:28336 to bitcoin.conf" : "Enable -zmqpubsequence in bitcoin.conf and set zmqSequenceEndpoint in config.toml to match";
+            if(Config.get().getServer().getBackendElectrumServer() != null) {
+                log.warn("Polling bitcoind every {}s with backendElectrumServer configured — clients may briefly display incorrect amounts for silent payments transactions. {}", pollSeconds, fix);
+            } else {
+                log.warn("Polling bitcoind every {}s, for more responsive updates {}", pollSeconds, fix);
             }
         }
     }
 
-    private synchronized void updateMempoolIndex() {
-        BitcoindClientService bitcoindService = getBitcoindService();
+    private synchronized void updateBlocksIndex() {
         HexFormat hexFormat = HexFormat.of();
 
-        Set<Sha256Hash> currentMempoolTxids = bitcoindService.getRawMempool();
-        Set<Sha256Hash> removedTxids = new HashSet<>(mempoolTxIds);
+        int startHeight = blocksIndex.getLastBlockIndexed() + 1;
+        int maxHeight = tip.height();
+        int totalBlocks = maxHeight - startHeight + 1;
+        long lastLogTime = System.currentTimeMillis();
+
+        for(int i = startHeight; i <= maxHeight; i++) {
+            if(useGetBlockVerbose) {
+                indexBlockVerbose(i, hexFormat);
+            } else {
+                indexBlockLegacy(i, hexFormat);
+            }
+
+            long now = System.currentTimeMillis();
+            if(now - lastLogTime >= 30_000) {
+                int blocksProcessed = i - startHeight + 1;
+                double percent = 100.0 * blocksProcessed / totalBlocks;
+                log.info("Indexing progress: {} / {} blocks ({}%, height {})", blocksProcessed, totalBlocks, String.format(Locale.ROOT, "%.1f", percent), i);
+                lastLogTime = now;
+            }
+        }
+    }
+
+    private void indexBlockVerbose(int height, HexFormat hexFormat) {
+        BitcoindClientService bitcoindService = getBitcoindService();
+        String blockHash = bitcoindService.getBlockHash(height);
+        if(height > tip.height() - MAX_REORG_DEPTH) {
+            recentBlocksMap.put(height, blockHash);
+        }
+
+        VerboseBlock vb = bitcoindService.getVerboseBlock(blockHash, 3);
+
+        Map<BlockTransaction, byte[]> eligibleTransactions = new LinkedHashMap<>();
+        for(VerboseBlock.VerboseTransaction vtx : vb.tx()) {
+            //fast path: populate the scriptPubKey cache and check the taproot-output gate directly from the JSON
+            Sha256Hash txid = Sha256Hash.wrap(vtx.txid());
+            boolean hasTaprootOutput = false;
+            for(VerboseBlock.VerboseVout vout : vtx.vout()) {
+                byte[] spkBytes = hexFormat.parseHex(vout.scriptPubKey().hex());
+                addtoScriptPubKeyCache(txid, vout.n(), spkBytes);
+                if(isP2tr(spkBytes)) {
+                    hasTaprootOutput = true;
+                }
+            }
+            boolean isCoinbase = !vtx.vin().isEmpty() && vtx.vin().getFirst().isCoinbase();
+            if(isCoinbase || !hasTaprootOutput) {
+                continue;
+            }
+
+            //eligible: parse the hex now so getInputPubKeys has witness / scriptSig data (not carried in v3 JSON)
+            Transaction tx = new Transaction(hexFormat.parseHex(vtx.hex()));
+            Map<HashIndex, Script> spentScriptPubKeys = new HashMap<>();
+            boolean missingPrevout = false;
+            for(int k = 0; k < vtx.vin().size(); k++) {
+                VerboseBlock.VerboseVin vin = vtx.vin().get(k);
+                if(vin.prevout() == null || vin.prevout().scriptPubKey() == null || vin.prevout().scriptPubKey().hex() == null) {
+                    missingPrevout = true;
+                    break;
+                }
+                TransactionInput in = tx.getInputs().get(k);
+                HashIndex hi = new HashIndex(in.getOutpoint().getHash(), in.getOutpoint().getIndex());
+                byte[] spkBytes = hexFormat.parseHex(vin.prevout().scriptPubKey().hex());
+                spentScriptPubKeys.put(hi, new Script(spkBytes));
+                addtoScriptPubKeyCache(hi.getHash(), (int)hi.getIndex(), spkBytes);
+            }
+
+            if(missingPrevout) {
+                log.warn("getblock verbosity=3 returned no prevout for an input at height {} - falling back to legacy per-input fetch for this block", height);
+                indexBlockLegacy(height, hexFormat);
+                return;
+            }
+
+            byte[] tweak = SilentPaymentUtils.getTweak(tx, spentScriptPubKeys, false);
+            if(tweak != null) {
+                BlockTransaction blkTx = new BlockTransaction(tx.getTxId(), height, new Date(vb.time() * 1000L), 0L, tx, Sha256Hash.wrap(vb.hash()));
+                eligibleTransactions.put(blkTx, SilentPaymentUtils.getSecp256k1PubKey(tweak));
+            }
+        }
+
+        blocksIndex.addToIndex(height, Sha256Hash.wrap(blockHash).getBytes(), eligibleTransactions);
+    }
+
+    private void indexBlockLegacy(int height, HexFormat hexFormat) {
+        BitcoindClientService bitcoindService = getBitcoindService();
+        String blockHash = bitcoindService.getBlockHash(height);
+        if(height > tip.height() - MAX_REORG_DEPTH) {
+            recentBlocksMap.put(height, blockHash);
+        }
+        String blockHex = (String)bitcoindService.getBlock(blockHash, 0);
+        Block block = new Block(hexFormat.parseHex(blockHex));
+
+        Map<BlockTransaction, byte[]> eligibleTransactions = new LinkedHashMap<>();
+        Map<HashIndex, Script> spentScriptPubKeys = new HashMap<>();
+        for(Transaction tx : block.getTransactions()) {
+            for(int outputIndex = 0; outputIndex < tx.getOutputs().size(); outputIndex++) {
+                byte[] scriptPubKeyBytes = tx.getOutputs().get(outputIndex).getScriptBytes();
+                addtoScriptPubKeyCache(tx.getTxId(), outputIndex, scriptPubKeyBytes);
+            }
+
+            if(!tx.isCoinBase() && containsTaprootOutput(tx)) {
+                for(TransactionInput txInput : tx.getInputs()) {
+                    HashIndex hashIndex = new HashIndex(txInput.getOutpoint().getHash(), txInput.getOutpoint().getIndex());
+                    spentScriptPubKeys.put(hashIndex, getScriptPubKey(bitcoindService, hexFormat, hashIndex));
+                }
+
+                byte[] tweak = SilentPaymentUtils.getTweak(tx, spentScriptPubKeys, false);
+                if(tweak != null) {
+                    BlockTransaction blkTx = new BlockTransaction(tx.getTxId(), height, block.getBlockHeader().getTimeAsDate(), 0L, tx, block.getHash());
+                    eligibleTransactions.put(blkTx, SilentPaymentUtils.getSecp256k1PubKey(tweak));
+                }
+            }
+        }
+
+        blocksIndex.addToIndex(height, Sha256Hash.wrap(blockHash).getBytes(), eligibleTransactions);
+    }
+
+    private synchronized void updateMempoolIndex() {
+        HexFormat hexFormat = HexFormat.of();
+
+        //snapshot before the RPC: if the ZMQ consumer removes a txid (R stream) after this point, this diff at worst re-issues a no-op removeFromIndex;
+        //if an RBF re-broadcast lands in getRawMempool() in the same window, the re-broadcast's own A event re-ingests it - the diff is not what re-indexes an RBF re-broadcast
+        Set<Sha256Hash> knownTxids = new HashSet<>(mempoolTxIds);
+        Set<Sha256Hash> currentMempoolTxids = getBitcoindService().getRawMempool();
+        Set<Sha256Hash> removedTxids = new HashSet<>(knownTxids);
         removedTxids.removeAll(currentMempoolTxids);
         Set<Sha256Hash> addedTxids = new HashSet<>(currentMempoolTxids);
-        addedTxids.removeAll(mempoolTxIds);
+        addedTxids.removeAll(knownTxids);
 
         Map<BlockTransaction, byte[]> eligibleTransactions = new LinkedHashMap<>();
         Map<HashIndex, Script> spentScriptPubKeys = new HashMap<>();
 
-        for(Sha256Hash addedTxid : addedTxids) {
-            try {
-                String txHex = (String)getBitcoindService().getRawTransaction(addedTxid.toString(), false);
-                Transaction tx = new Transaction(hexFormat.parseHex(txHex));
-                for(int outputIndex = 0; outputIndex < tx.getOutputs().size(); outputIndex++) {
-                    byte[] scriptPubKeyBytes = tx.getOutputs().get(outputIndex).getScriptBytes();
-                    addtoScriptPubKeyCache(tx.getTxId(), outputIndex, scriptPubKeyBytes);
+        try {
+            Map<Sha256Hash, String> hexByTxid = fetchRawTxBatch(addedTxids);
+            for(Sha256Hash addedTxid : addedTxids) {
+                String txHex = hexByTxid.get(addedTxid);
+                if(txHex == null) {
+                    continue;
                 }
-
-                if(!tx.isCoinBase() && containsTaprootOutput(tx)) {
-                    for(TransactionInput txInput : tx.getInputs()) {
-                        HashIndex hashIndex = new HashIndex(txInput.getOutpoint().getHash(), txInput.getOutpoint().getIndex());
-                        spentScriptPubKeys.put(hashIndex, getScriptPubKey(bitcoindService, hexFormat, hashIndex));
-                    }
-
-                    byte[] tweak = SilentPaymentUtils.getTweak(tx, spentScriptPubKeys);
-                    if(tweak != null) {
-                        BlockTransaction blkTx = new BlockTransaction(tx.getTxId(), 0, null, 0L, tx, null);
-                        eligibleTransactions.put(blkTx, tweak);
-                    }
-                }
-            } catch(JsonRpcException e) {
-                //ignore, transaction removed from mempool
+                ingestMempoolTxFromHex(addedTxid, txHex, spentScriptPubKeys, eligibleTransactions, hexFormat);
             }
-        }
 
-        if(!removedTxids.isEmpty()) {
-            mempoolIndex.removeFromIndex(removedTxids);
-        }
-        if(!eligibleTransactions.isEmpty()) {
-            mempoolIndex.addToIndex(eligibleTransactions);
+            if(!removedTxids.isEmpty()) {
+                mempoolIndex.removeFromIndex(removedTxids);
+            }
+            if(!eligibleTransactions.isEmpty()) {
+                mempoolIndex.addToIndex(0, null, eligibleTransactions);
+            }
+        } catch(RuntimeException e) {
+            //nothing was committed to the index - drop the would-be-indexed txids so a later diff retries them
+            eligibleTransactions.keySet().forEach(blkTx -> mempoolTxIds.remove(blkTx.getHash()));
+            throw e;
         }
 
         mempoolTxIds.removeAll(removedTxids);
-        mempoolTxIds.addAll(addedTxids);
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<Sha256Hash, String> fetchRawTxBatch(Collection<Sha256Hash> txids) {
+        if(txids.isEmpty()) {
+            return Map.of();
+        }
+        PagedBatchRequestBuilder<Sha256Hash, String> builder = PagedBatchRequestBuilder.create(bitcoindTransport, rpcIdCounter)
+                .keysType(Sha256Hash.class).returnType(String.class);
+        for(Sha256Hash txid : txids) {
+            builder.add(txid, "getrawtransaction", txid.toString(), false);
+        }
+        try {
+            return builder.execute();
+        } catch(JsonRpcBatchException e) {
+            return (Map<Sha256Hash, String>)e.getSuccesses();
+        } catch(Exception e) {
+            if(e instanceof RuntimeException re) {
+                throw re;
+            }
+            throw new RuntimeException("Error executing batched getrawtransaction request", e);
+        }
+    }
+
+    private void ingestMempoolTx(Sha256Hash txid, Map<HashIndex, Script> spentScriptPubKeys, Map<BlockTransaction, byte[]> eligibleTransactions, HexFormat hexFormat) {
+        String txHex;
+        try {
+            txHex = (String)getBitcoindService().getRawTransaction(txid.toString(), false);
+        } catch(JsonRpcException e) {
+            //transaction removed from mempool before we could fetch it
+            return;
+        }
+
+        ingestMempoolTxFromHex(txid, txHex, spentScriptPubKeys, eligibleTransactions, hexFormat);
+    }
+
+    /**
+     * Ingest an already-fetched mempool transaction. Idempotent — gated on an atomic insert into {@link #mempoolTxIds},
+     * so a txid already ingested (via ZMQ or the safety-net diff) is skipped.
+     */
+    private void ingestMempoolTxFromHex(Sha256Hash txid, String txHex, Map<HashIndex, Script> spentScriptPubKeys, Map<BlockTransaction, byte[]> eligibleTransactions, HexFormat hexFormat) {
+        if(!mempoolTxIds.add(txid)) {
+            return;
+        }
+
+        try {
+            Transaction tx = new Transaction(hexFormat.parseHex(txHex));
+            for(int outputIndex = 0; outputIndex < tx.getOutputs().size(); outputIndex++) {
+                byte[] scriptPubKeyBytes = tx.getOutputs().get(outputIndex).getScriptBytes();
+                addtoScriptPubKeyCache(tx.getTxId(), outputIndex, scriptPubKeyBytes);
+            }
+
+            if(!tx.isCoinBase() && containsTaprootOutput(tx)) {
+                BitcoindClientService bitcoindService = getBitcoindService();
+                for(TransactionInput txInput : tx.getInputs()) {
+                    HashIndex hashIndex = new HashIndex(txInput.getOutpoint().getHash(), txInput.getOutpoint().getIndex());
+                    spentScriptPubKeys.put(hashIndex, getScriptPubKey(bitcoindService, hexFormat, hashIndex));
+                }
+
+                byte[] tweak = SilentPaymentUtils.getTweak(tx, spentScriptPubKeys, false);
+                if(tweak != null) {
+                    BlockTransaction blkTx = new BlockTransaction(tx.getTxId(), 0, null, 0L, tx, null);
+                    eligibleTransactions.put(blkTx, SilentPaymentUtils.getSecp256k1PubKey(tweak));
+                }
+            }
+        } catch(RuntimeException e) {
+            //transient failure - drop the txid so a later diff re-ingests it
+            mempoolTxIds.remove(txid);
+            throw e;
+        }
+    }
+
+    private boolean isLoopbackBitcoind() {
+        Server coreServer = Config.get().getCore().getServerObj();
+        String host = coreServer != null ? coreServer.getHost() : "127.0.0.1";
+        try {
+            return InetAddress.getByName(host).isLoopbackAddress();
+        } catch(UnknownHostException e) {
+            return false;
+        }
+    }
+
+    private ZmqDiscovery discoverZmqSequenceEndpoint() {
+        try {
+            for(ZmqNotification notification : getBitcoindService().getZmqNotifications()) {
+                if("pubsequence".equals(notification.type()) && notification.address() != null) {
+                    return new ZmqDiscovery(normaliseZmqAddress(notification.address()), false);
+                }
+            }
+        } catch(JsonRpcException e) {
+            if(e.getErrorMessage() != null && e.getErrorMessage().getCode() == -32601) {
+                return new ZmqDiscovery(null, true);
+            }
+            log.debug("Could not auto-discover ZMQ endpoint from Bitcoin Core", e);
+        } catch(Exception e) {
+            log.debug("Could not auto-discover ZMQ endpoint from Bitcoin Core", e);
+        }
+
+        return new ZmqDiscovery(null, false);
+    }
+
+    private static String normaliseZmqAddress(String address) {
+        //wildcard binds aren't usable as connect targets; substitute loopback
+        if(address.startsWith("tcp://0.0.0.0:")) {
+            return "tcp://127.0.0.1:" + address.substring("tcp://0.0.0.0:".length());
+        }
+        if(address.startsWith("tcp://[::]:")) {
+            return "tcp://[::1]:" + address.substring("tcp://[::]:".length());
+        }
+
+        return address;
+    }
+
+    private void startZmqSequenceSubscriber(String endpoint) {
+        if(stopped) {
+            return;
+        }
+
+        zmqContext = new ZContext();
+        zmqSubscriberThread = Thread.ofPlatform().daemon().name("BitcoindZmqSequence").start(() -> {
+            try(ZMQ.Socket socket = zmqContext.createSocket(SocketType.SUB)) {
+                socket.setReconnectIVL(100);
+                socket.setReconnectIVLMax(10_000);
+                socket.subscribe("sequence");
+                socket.connect(endpoint);
+                log.info("Subscribed to ZMQ sequence publisher at {}", endpoint);
+
+                while(!stopped && !Thread.currentThread().isInterrupted()) {
+                    String topic = socket.recvStr();
+                    byte[] body = socket.hasReceiveMore() ? socket.recv() : null;
+                    while(socket.hasReceiveMore()) {
+                        socket.recv();  //drain remaining frames (the per-topic sequence number, plus any added in future bitcoind versions)
+                    }
+                    if(!"sequence".equals(topic) || body == null || body.length < 33) {
+                        continue;
+                    }
+                    lastZmqMessageMs = System.currentTimeMillis();
+
+                    char label = (char)body[32];
+                    if(label == 'A' || label == 'R') {
+                        Sha256Hash txid = Sha256Hash.wrap(Arrays.copyOf(body, 32));
+                        if(!zmqQueue.offer(new MempoolSeqEvent(txid, label == 'R'))) {
+                            log.warn("ZMQ mempool queue full, dropping {} for txid {} (safety-net diff will recover it)", label, txid);
+                        }
+                    } else if(label == 'C' || label == 'D') {
+                        //block connect/disconnect: don't ingest here, just kick an immediate PollTask run - reorg detection, tip update and the forced mempool diff all live there
+                        //pollPending coalesces bursts (testnet3 difficulty-reset storms, multi-block reorgs, post-outage catch-up) down to ~1 in-flight + 1 queued
+                        if(!stopped && pollPending.compareAndSet(false, true)) {
+                            timer.schedule(new PollTask(), 0);
+                        }
+                    }
+                    //'R' fires for non-block removals only (RBF/eviction/expiry); mined txs produce 'C' (block connect) with no per-tx 'R'
+                }
+            } catch(Throwable t) {
+                if(!stopped) {
+                    log.error("Bitcoin Core ZMQ sequence subscriber exited", t);
+                }
+            }
+        });
+
+        zmqConsumerThread = Thread.ofPlatform().daemon().name("BitcoindZmqConsumer").start(this::zmqConsumerLoop);
+
+        timer.schedule(new TimerTask() {
+            @Override
+            public void run() {
+                if(!stopped && lastZmqMessageMs == 0L && Network.get() == Network.MAINNET) {
+                    log.warn("No ZMQ messages received from Bitcoin Core within 60s at {} - verify -zmqpubsequence is configured on this endpoint. " +
+                            "Mempool ingestion latency will be up to {}s until ZMQ messages arrive", endpoint, ZMQ_MEMPOOL_DIFF_INTERVAL_MS / 1000);
+                }
+            }
+        }, 60_000);
+    }
+
+    private void zmqConsumerLoop() {
+        Map<BlockTransaction, byte[]> eligibleTransactions = new LinkedHashMap<>();
+        Set<Sha256Hash> removedTxids = new HashSet<>();
+        Map<HashIndex, Script> spentScriptPubKeys = new HashMap<>();
+        HexFormat hexFormat = HexFormat.of();
+
+        while(!stopped && !Thread.currentThread().isInterrupted()) {
+            try {
+                MempoolSeqEvent first = zmqQueue.poll(1, TimeUnit.SECONDS);
+                if(first == null) {
+                    continue;
+                }
+
+                long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(FLUSH_DEBOUNCE_MS);
+                processSeqEvent(first, spentScriptPubKeys, eligibleTransactions, removedTxids, hexFormat);
+                while(eligibleTransactions.size() + removedTxids.size() < FLUSH_BATCH_SIZE) {
+                    long remaining = deadline - System.nanoTime();
+                    if(remaining <= 0) {
+                        break;
+                    }
+                    MempoolSeqEvent next = zmqQueue.poll(remaining, TimeUnit.NANOSECONDS);
+                    if(next == null) {
+                        break;
+                    }
+                    processSeqEvent(next, spentScriptPubKeys, eligibleTransactions, removedTxids, hexFormat);
+                }
+
+                //add before remove: if a txid somehow ended up in both batches (it shouldn't - see processSeqEvent), the net result is "absent", correct for an A-then-R
+                if(!eligibleTransactions.isEmpty()) {
+                    mempoolIndex.addToIndex(0, null, eligibleTransactions);
+                    eligibleTransactions.clear();
+                }
+                if(!removedTxids.isEmpty()) {
+                    mempoolIndex.removeFromIndex(removedTxids);
+                    removedTxids.clear();
+                }
+                spentScriptPubKeys.clear();
+            } catch(InterruptedException e) {
+                return;
+            } catch(Throwable t) {
+                log.error("Error processing ZMQ mempool transactions", t);
+                eligibleTransactions.keySet().forEach(blkTx -> mempoolTxIds.remove(blkTx.getHash()));
+                eligibleTransactions.clear();
+                mempoolTxIds.addAll(removedTxids);
+                removedTxids.clear();
+                spentScriptPubKeys.clear();
+            }
+        }
+    }
+
+    private void processSeqEvent(MempoolSeqEvent event, Map<HashIndex, Script> spentScriptPubKeys, Map<BlockTransaction, byte[]> eligibleTransactions, Set<Sha256Hash> removedTxids, HexFormat hexFormat) {
+        if(event.removed()) {
+            if(mempoolTxIds.remove(event.txid())) {
+                removedTxids.add(event.txid());
+                eligibleTransactions.keySet().removeIf(blkTx -> blkTx.getHash().equals(event.txid()));
+            }
+        } else {
+            ingestMempoolTx(event.txid(), spentScriptPubKeys, eligibleTransactions, hexFormat);
+        }
     }
 
     public void stop() {
         timer.cancel();
         stopped = true;
+        if(zmqSubscriberThread != null) {
+            zmqSubscriberThread.interrupt();
+        }
+        if(zmqConsumerThread != null) {
+            zmqConsumerThread.interrupt();
+        }
+        if(zmqContext != null) {
+            zmqContext.close();
+        }
     }
 
     public BitcoindClientService getBitcoindService() {
@@ -243,11 +640,16 @@ public class BitcoindClient {
     private Script getScriptPubKey(BitcoindClientService bitcoindClientService, HexFormat hexFormat, HashIndex hashIndex) {
         Script scriptPubKey = getFromScriptPubKeyCache(hashIndex);
         if(scriptPubKey == null) {
-            String txHex = (String)bitcoindClientService.getRawTransaction(hashIndex.getHash().toString(), false);
-            Transaction tx = new Transaction(hexFormat.parseHex(txHex));
-            TransactionOutput txOutput = tx.getOutputs().get((int)hashIndex.getIndex());
-            addtoScriptPubKeyCache(hashIndex.getHash(), (int)hashIndex.getIndex(), txOutput.getScriptBytes());
-            scriptPubKey = getFromScriptPubKeyCache(hashIndex);
+            try {
+                String txHex = (String)bitcoindClientService.getRawTransaction(hashIndex.getHash().toString(), false);
+                Transaction tx = new Transaction(hexFormat.parseHex(txHex));
+                TransactionOutput txOutput = tx.getOutputs().get((int)hashIndex.getIndex());
+                addtoScriptPubKeyCache(hashIndex.getHash(), (int)hashIndex.getIndex(), txOutput.getScriptBytes());
+                scriptPubKey = getFromScriptPubKeyCache(hashIndex);
+            } catch(Exception e) {
+                log.error("Error retrieving scriptPubKey for txid " + hashIndex.getHash() + " output index " + hashIndex.getIndex(), e);
+                throw e;
+            }
         }
 
         return scriptPubKey;
@@ -256,11 +658,15 @@ public class BitcoindClient {
     private class PollTask extends TimerTask {
         @Override
         public void run() {
+            //clear at the start: a 'C' arriving while this run is in flight re-arms pollPending and schedules exactly one follow-up poll (which picks up anything connected during this run)
+            pollPending.set(false);
+
             if(stopped) {
                 timer.cancel();
             }
 
             try {
+                boolean newBlock = false;
                 if(syncing) {
                     BlockchainInfo blockchainInfo = getBitcoindService().getBlockchainInfo();
                     if(blockchainInfo.initialblockdownload() && !isEmptyBlockchain(blockchainInfo)) {
@@ -288,18 +694,20 @@ public class BitcoindClient {
                             }
                         }
 
-                        int blocksReorged = tip.height() - reorgStartHeight + 1;
+                        int blocksReorged = tip.height() - reorgStartHeight;
                         if(blocksReorged > 1) {
-                            log.info("Reorg detected of last block, block height " + tip.height() + " was " + lastBlock + " and now is " + blockhash);
-                        } else {
                             log.info("Reorg detected of last " + blocksReorged + " blocks, block height " + tip.height() + " was " + lastBlock + " and now is " + blockhash);
+                        } else {
+                            log.info("Reorg detected of last block, block height " + tip.height() + " was " + lastBlock + " and now is " + blockhash);
                         }
 
-                        Frigate.getEventBus().post(new BlockReorgEvent(reorgStartHeight));
-                        blocksIndex.removeFromIndex(reorgStartHeight);
+                        Frigate.getEventBus().post(new BlockReorgSyncStart(reorgStartHeight));
+                        blocksIndex.removeFromIndex(reorgStartHeight + 1);
                         updateBlocksIndex();
+                        Frigate.getEventBus().post(new BlockReorgSyncComplete(reorgStartHeight));
 
                         lastBlock = null;
+                        newBlock = true;
                     }
                 }
 
@@ -309,12 +717,19 @@ public class BitcoindClient {
                 if(currentBlock == null || !currentBlock.equals(blockchainInfo.bestblockhash())) {
                     VerboseBlockHeader blockHeader = getBitcoindService().getBlockHeader(blockchainInfo.bestblockhash());
                     tip = blockHeader.getBlockHeader();
-                    log.info("New block height " + tip.height());
+                    log.debug("New block height " + tip.height());
                     Frigate.getEventBus().post(tip);
                     updateBlocksIndex();
+                    newBlock = true;
                 }
 
-                updateMempoolIndex();
+                boolean zmqHealthy = zmqContext != null && lastZmqMessageMs != 0L && System.currentTimeMillis() - lastZmqMessageMs < ZMQ_STALENESS_THRESHOLD_MS;
+                long mempoolDiffInterval = zmqHealthy ? ZMQ_MEMPOOL_DIFF_INTERVAL_MS : POLL_MEMPOOL_DIFF_INTERVAL_MS;
+                //force the diff when a block was just connected/reorged: a batch of txids left the mempool at once (mined txs get no per-tx 'R'), evict them now rather than up to ZMQ_MEMPOOL_DIFF_INTERVAL_MS later
+                if(newBlock || System.currentTimeMillis() - lastMempoolDiffMs >= mempoolDiffInterval) {
+                    updateMempoolIndex();
+                    lastMempoolDiffMs = System.currentTimeMillis();
+                }
 
                 lastBlock = blockchainInfo.bestblockhash();
             } catch(Exception e) {
@@ -431,15 +846,20 @@ public class BitcoindClient {
         }
 
         // P2TR: 34 bytes - OP_1 <32-byte taproot output>
-        if(length == 34 &&
-                scriptPubKey[0] == (byte) 0x51 &&  // OP_1
-                scriptPubKey[1] == (byte) 0x20) {  // Push 32 bytes
+        if(isP2tr(scriptPubKey)) {
             return ScriptType.P2TR;
         }
 
         return null;
     }
 
+    private static boolean isP2tr(byte[] scriptPubKey) {
+        return scriptPubKey != null
+                && scriptPubKey.length == 34
+                && scriptPubKey[0] == (byte) 0x51   // OP_1
+                && scriptPubKey[1] == (byte) 0x20;  // Push 32 bytes
+    }
+
     private static File getDefaultCoreDataDir() {
         OsType osType = OsType.getCurrent();
         if(osType == OsType.MACOS) {
@@ -460,4 +880,12 @@ public class BitcoindClient {
             }
         };
     }
+
+    public boolean containsSubmitPackage() {
+        return networkInfo.version() >= MIN_SUBMIT_PACKAGE_VERSION;
+    }
+
+    private record MempoolSeqEvent(Sha256Hash txid, boolean removed) {}
+
+    private record ZmqDiscovery(String endpoint, boolean unsupported) {}
 }

src/main/java/com/sparrowwallet/frigate/bitcoind/BitcoindClientService.java

@@ -8,6 +8,7 @@ import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam;
 import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService;
 import com.sparrowwallet.drongo.protocol.Sha256Hash;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -20,8 +21,11 @@ public interface BitcoindClientService {
     @JsonRpcMethod("getnetworkinfo")
     NetworkInfo getNetworkInfo();
 
+    @JsonRpcMethod("getzmqnotifications")
+    List<ZmqNotification> getZmqNotifications();
+
     @JsonRpcMethod("estimatesmartfee")
-    FeeInfo estimateSmartFee(@JsonRpcParam("conf_target") int blocks);
+    FeeInfo estimateSmartFee(@JsonRpcParam("conf_target") int blocks, @JsonRpcParam("estimate_mode") @JsonRpcOptional String mode);
 
     @JsonRpcMethod("getrawmempool")
     Set<Sha256Hash> getRawMempool();
@@ -50,6 +54,9 @@ public interface BitcoindClientService {
     @JsonRpcMethod("getblock")
     Object getBlock(@JsonRpcParam("blockhash") String blockhash, @JsonRpcOptional @JsonRpcParam("verbosity") int verbosity);
 
+    @JsonRpcMethod("getblock")
+    VerboseBlock getVerboseBlock(@JsonRpcParam("blockhash") String blockhash, @JsonRpcParam("verbosity") int verbosity);
+
     @JsonRpcMethod("getrawtransaction")
     Object getRawTransaction(@JsonRpcParam("txid") String txid, @JsonRpcParam("verbose") boolean verbose);
 
@@ -61,5 +68,8 @@ public interface BitcoindClientService {
 
     @JsonRpcMethod("sendrawtransaction")
     String sendRawTransaction(@JsonRpcParam("hexstring") String rawTx, @JsonRpcParam("maxfeerate") Double maxFeeRate);
+
+    @JsonRpcMethod("submitpackage")
+    PackageResult submitPackage(@JsonRpcParam("package") String[] rawTxes, @JsonRpcParam("maxfeerate") @JsonRpcOptional Double maxFeeRate, @JsonRpcParam("maxburnamount") @JsonRpcOptional Double maxBurnAmount);
 }
 

src/main/java/com/sparrowwallet/frigate/bitcoind/BitcoindTransport.java

@@ -2,24 +2,27 @@ package com.sparrowwallet.frigate.bitcoind;
 
 import com.github.arteam.simplejsonrpc.client.Transport;
 import com.sparrowwallet.drongo.Network;
+import com.sparrowwallet.frigate.io.Config;
 import com.sparrowwallet.frigate.io.Server;
+import com.sparrowwallet.frigate.io.SslUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.net.ssl.*;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
 import java.io.*;
 import java.net.*;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
 import java.util.Base64;
 
 public class BitcoindTransport implements Transport {
     private static final Logger log = LoggerFactory.getLogger(BitcoindTransport.class);
     public static final String COOKIE_FILENAME = ".cookie";
 
+    private static final int CONNECT_TIMEOUT_MS = 10_000;
+
     private final Server bitcoindServer;
     private URL bitcoindUrl;
     private File cookieFile;
@@ -54,12 +57,15 @@ public class BitcoindTransport implements Transport {
         HttpURLConnection connection = (HttpURLConnection)bitcoindUrl.openConnection();
 
         if(connection instanceof HttpsURLConnection httpsURLConnection) {
-            SSLSocketFactory sslSocketFactory = getTrustAllSocketFactory();
+            SSLSocketFactory sslSocketFactory = SslUtil.getTrustAllSocketFactory();
             if(sslSocketFactory != null) {
                 httpsURLConnection.setSSLSocketFactory(sslSocketFactory);
             }
         }
 
+        connection.setConnectTimeout(CONNECT_TIMEOUT_MS);
+        connection.setReadTimeout(Config.get().getCore().getRpcRequestTimeoutMillis());
+
         connection.setRequestMethod("POST");
         connection.setRequestProperty("Content-Type", "application/json");
 
@@ -135,29 +141,4 @@ public class BitcoindTransport implements Transport {
         return bitcoindDir;
     }
 
-    private SSLSocketFactory getTrustAllSocketFactory() {
-        TrustManager[] trustAllCerts = new TrustManager[] {
-                new X509TrustManager() {
-                    public X509Certificate[] getAcceptedIssuers() {
-                        return new X509Certificate[0];
-                    }
-
-                    public void checkClientTrusted(X509Certificate[] certs, String authType) throws CertificateException {
-                    }
-
-                    public void checkServerTrusted(X509Certificate[] certs, String authType) throws CertificateException {
-                    }
-                }
-        };
-
-        try {
-            SSLContext sslContext = SSLContext.getInstance("TLS");
-            sslContext.init(null, trustAllCerts, null);
-            return sslContext.getSocketFactory();
-        } catch (Exception e) {
-            log.error("Error creating SSL socket factory", e);
-        }
-
-        return null;
-    }
 }

src/main/java/com/sparrowwallet/frigate/bitcoind/BlockReorgEvent.java

@@ -1,3 +0,0 @@
-package com.sparrowwallet.frigate.bitcoind;
-
-public record BlockReorgEvent(int startHeight) {}

src/main/java/com/sparrowwallet/frigate/bitcoind/BlockReorgSyncComplete.java

@@ -0,0 +1,3 @@
+package com.sparrowwallet.frigate.bitcoind;
+
+public record BlockReorgSyncComplete(int reorgStartHeight) {}

src/main/java/com/sparrowwallet/frigate/bitcoind/BlockReorgSyncStart.java

@@ -0,0 +1,3 @@
+package com.sparrowwallet.frigate.bitcoind;
+
+public record BlockReorgSyncStart(int reorgStartHeight) {}

src/main/java/com/sparrowwallet/frigate/bitcoind/MempoolEntry.java

@@ -1,7 +1,7 @@
 package com.sparrowwallet.frigate.bitcoind;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.sparrowwallet.frigate.index.TxEntry;
+import com.sparrowwallet.frigate.electrum.TxEntry;
 
 @JsonIgnoreProperties(ignoreUnknown = true)
 public record MempoolEntry(int vsize, int ancestorsize, boolean bip125_replaceable, FeesMempoolEntry fees) {

src/main/java/com/sparrowwallet/frigate/bitcoind/MempoolInfo.java

@@ -1,7 +1,10 @@
 package com.sparrowwallet.frigate.bitcoind;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.sparrowwallet.drongo.protocol.Transaction;
 
 @JsonIgnoreProperties(ignoreUnknown = true)
-public record MempoolInfo(double minrelaytxfee) {
+public record MempoolInfo(double mempoolminfee, double minrelaytxfee, double incrementalrelayfee) {
+    public static final double DEFAULT_FEE_RATE = (Transaction.DEFAULT_MIN_RELAY_FEE / Transaction.SATOSHIS_PER_BITCOIN) * 1000;
+    public static final MempoolInfo DEFAULT = new MempoolInfo(DEFAULT_FEE_RATE, DEFAULT_FEE_RATE, DEFAULT_FEE_RATE);
 }

src/main/java/com/sparrowwallet/frigate/bitcoind/PackageResult.java

@@ -0,0 +1,11 @@
+package com.sparrowwallet.frigate.bitcoind;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+public record PackageResult(String package_msg, @JsonProperty("tx-results") Map<String, TxResult> txResults, @JsonProperty("replaced-transactions") List<String> replacedTransactions) {
+    public record TxResult(String txid, @JsonProperty("other-wtxid") String otherWtxid, Integer vsize, Fees fees, String error) {}
+    public record Fees(Double base, @JsonProperty("effective-feerate") Double effectiveFeerate, @JsonProperty("effective-includes") List<String> effectiveIncludes) {}
+}

src/main/java/com/sparrowwallet/frigate/bitcoind/PagedBatchRequestBuilder.java

@@ -0,0 +1,150 @@
+package com.sparrowwallet.frigate.bitcoind;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.arteam.simplejsonrpc.client.JsonRpcClient;
+import com.github.arteam.simplejsonrpc.client.Transport;
+import com.github.arteam.simplejsonrpc.client.builder.AbstractBuilder;
+import com.github.arteam.simplejsonrpc.client.builder.BatchRequestBuilder;
+import com.github.arteam.simplejsonrpc.client.exception.JsonRpcBatchException;
+import com.github.arteam.simplejsonrpc.core.domain.ErrorMessage;
+import com.google.common.collect.Lists;
+import com.sparrowwallet.frigate.io.Config;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class PagedBatchRequestBuilder<K, V> extends AbstractBuilder {
+    public static final int DEFAULT_MAX_ATTEMPTS = 3;
+    public static final int RETRY_DELAY_SECS = 1;
+
+    private final AtomicLong counter;
+
+    private final List<Request<K>> requests;
+
+    /**
+     * Type of request ids
+     */
+    private final Class<K> keysType;
+
+    /**
+     * Expected return type for all requests
+     */
+    private final Class<V> returnType;
+
+    public PagedBatchRequestBuilder(Transport transport, ObjectMapper mapper, AtomicLong counter) {
+        this(transport, mapper, new ArrayList<>(), null, null, counter);
+    }
+
+    public PagedBatchRequestBuilder(Transport transport, ObjectMapper mapper,
+                                    List<Request<K>> requests,
+                                    Class<K> keysType, Class<V> returnType,
+                                    AtomicLong counter) {
+        super(transport, mapper);
+        this.requests = requests;
+        this.keysType = keysType;
+        this.returnType = returnType;
+        this.counter = counter;
+    }
+
+    /**
+     * Adds a new request without specifying a return type
+     */
+    public PagedBatchRequestBuilder<K, V> add(K id, String method, Object... params) {
+        requests.add(new Request<>(id, counter == null ? null : counter.incrementAndGet(), method, params));
+        return this;
+    }
+
+    /**
+     * Sets type of request keys.
+     */
+    public <NK> PagedBatchRequestBuilder<NK, V> keysType(Class<NK> keysClass) {
+        return new PagedBatchRequestBuilder<>(transport, mapper, new ArrayList<>(), keysClass, returnType, counter);
+    }
+
+    /**
+     * Sets an expected response type of requests.
+     */
+    public <NV> PagedBatchRequestBuilder<K, NV> returnType(Class<NV> valuesClass) {
+        return new PagedBatchRequestBuilder<>(transport, mapper, requests, keysType, valuesClass, counter);
+    }
+
+    public Map<K, V> execute() throws Exception {
+        return execute(DEFAULT_MAX_ATTEMPTS);
+    }
+
+    /**
+     * Validates, executes the request and processes the response
+     */
+    public Map<K, V> execute(int maxAttempts) throws Exception {
+        Map<K, V> allResults = new HashMap<>();
+        JsonRpcClient client = new JsonRpcClient(transport);
+
+        List<List<Request<K>>> pages = Lists.partition(requests, getPageSize());
+        for(List<Request<K>> page : pages) {
+            if(counter != null) {
+                Map<Long, K> counterIdMap = new HashMap<>();
+                BatchRequestBuilder<Long, V> batchRequest = client.createBatchRequest().keysType(Long.class).returnType(returnType);
+                for(Request<K> request : page) {
+                    counterIdMap.put(request.counterId(), request.id());
+                    batchRequest.add(request.counterId(), request.method(), request.params());
+                }
+
+                try {
+                    Map<Long, V> pageResult = new RetryLogic<Map<Long, V>>(maxAttempts, RETRY_DELAY_SECS, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute);
+                    for(Map.Entry<Long, V> pageEntry : pageResult.entrySet()) {
+                        allResults.put(counterIdMap.get(pageEntry.getKey()), pageEntry.getValue());
+                    }
+                } catch(JsonRpcBatchException e) {
+                    Map<Object, Object> mappedSuccesses = new HashMap<>();
+                    for(Map.Entry<?, ?> successEntry : e.getSuccesses().entrySet()) {
+                        mappedSuccesses.put(counterIdMap.get((Long)successEntry.getKey()), successEntry.getValue());
+                    }
+                    Map<Object, ErrorMessage> mappedErrors = new HashMap<>();
+                    for(Map.Entry<?, ErrorMessage> errorEntry : e.getErrors().entrySet()) {
+                        mappedErrors.put(counterIdMap.get((Long)errorEntry.getKey()), errorEntry.getValue());
+                    }
+                    throw new JsonRpcBatchException(e.getMessage(), mappedSuccesses, mappedErrors);
+                }
+            } else {
+                BatchRequestBuilder<K, V> batchRequest = client.createBatchRequest().keysType(keysType).returnType(returnType);
+                for(Request<K> request : page) {
+                    if(request.id() instanceof String strReq) {
+                        batchRequest.add(strReq, request.method(), request.params());
+                    } else if(request.id() instanceof Integer intReq) {
+                        batchRequest.add(intReq, request.method(), request.params());
+                    } else {
+                        throw new IllegalArgumentException("Id of class " + request.id().getClass().getName() + " not supported");
+                    }
+                }
+
+                Map<K, V> pageResult = new RetryLogic<Map<K, V>>(maxAttempts, RETRY_DELAY_SECS, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute);
+                allResults.putAll(pageResult);
+            }
+        }
+
+        return allResults;
+    }
+
+    private int getPageSize() {
+        return Config.get().getCore().getRpcBatchSizeValue();
+    }
+
+    /**
+     * Creates a builder of a JSON-RPC batch request in initial state
+     */
+    public static PagedBatchRequestBuilder<?, ?> create(Transport transport) {
+        return new PagedBatchRequestBuilder<>(transport, new ObjectMapper(), null);
+    }
+
+    /**
+     * Creates a builder of a JSON-RPC batch request in initial state with a counter for request ids
+     */
+    public static PagedBatchRequestBuilder<?, ?> create(Transport transport, AtomicLong counter) {
+        return new PagedBatchRequestBuilder<>(transport, new ObjectMapper(), counter);
+    }
+
+    private record Request<K>(K id, Long counterId, String method, Object[] params) {}
+}

src/main/java/com/sparrowwallet/frigate/bitcoind/RetryLogic.java

@@ -0,0 +1,56 @@
+package com.sparrowwallet.frigate.bitcoind;
+
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Generic retry logic. Delegate must throw the specified exception type to trigger the retry logic.
+ */
+public class RetryLogic<T> {
+    public static interface Delegate<T> {
+        T call() throws Exception;
+    }
+
+    private final int maxAttempts;
+    private final int retryWaitSeconds;
+    @SuppressWarnings("rawtypes")
+    private final List<Class> retryExceptionTypes;
+
+    public RetryLogic(int maxAttempts, int retryWaitSeconds, @SuppressWarnings("rawtypes") Class retryExceptionType) {
+        this(maxAttempts, retryWaitSeconds, List.of(retryExceptionType));
+    }
+
+    public RetryLogic(int maxAttempts, int retryWaitSeconds, @SuppressWarnings("rawtypes") List<Class> retryExceptionTypes) {
+        this.maxAttempts = maxAttempts;
+        this.retryWaitSeconds = Math.max(retryWaitSeconds, 1);
+        this.retryExceptionTypes = retryExceptionTypes;
+    }
+
+    public T getResult(Delegate<T> caller) throws Exception {
+        T result = null;
+        int remainingAttempts = maxAttempts;
+        do {
+            try {
+                return caller.call();
+            } catch(Exception e) {
+                if(retryExceptionTypes.contains(e.getClass())) {
+                    if(--remainingAttempts == 0) {
+                        throw new ServerException("Retries exhausted", e);
+                    } else {
+                        try {
+                            //Sleep with a +/- 2 seconds random wait time to avoid simultaneous retries
+                            Thread.sleep((1000L * (retryWaitSeconds - 1)) + new Random().nextInt(2000));
+                        } catch(InterruptedException ie) {
+                            //ignore
+                        }
+                    }
+                } else {
+                    throw e;
+                }
+            }
+
+        } while(remainingAttempts > 0);
+
+        throw new IllegalStateException("Should be impossible");
+    }
+}

src/main/java/com/sparrowwallet/frigate/bitcoind/ServerException.java

@@ -0,0 +1,19 @@
+package com.sparrowwallet.frigate.bitcoind;
+
+public class ServerException extends Exception {
+    public ServerException() {
+        super();
+    }
+
+    public ServerException(String message) {
+        super(message);
+    }
+
+    public ServerException(Throwable cause) {
+        super(cause);
+    }
+
+    public ServerException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

src/main/java/com/sparrowwallet/frigate/bitcoind/VerboseBlock.java

@@ -0,0 +1,28 @@
+package com.sparrowwallet.frigate.bitcoind;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public record VerboseBlock(String hash, int height, long time, List<VerboseTransaction> tx) {
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public record VerboseTransaction(String txid, @JsonProperty("hex") String hex, List<VerboseVin> vin, List<VerboseVout> vout) {}
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public record VerboseVin(String txid, @JsonProperty("vout") Integer voutIndex, VerbosePrevout prevout) {
+        public boolean isCoinbase() {
+            return txid == null;
+        }
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public record VerbosePrevout(@JsonProperty("scriptPubKey") VerboseScriptPubKey scriptPubKey) {}
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public record VerboseVout(int n, @JsonProperty("scriptPubKey") VerboseScriptPubKey scriptPubKey) {}
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public record VerboseScriptPubKey(String hex) {}
+}

src/main/java/com/sparrowwallet/frigate/bitcoind/ZmqNotification.java

@@ -0,0 +1,6 @@
+package com.sparrowwallet.frigate.bitcoind;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public record ZmqNotification(String type, String address, long hwm) {}

src/main/java/com/sparrowwallet/frigate/cli/Args.java

@@ -1,6 +1,8 @@
 package com.sparrowwallet.frigate.cli;
 
+import com.beust.jcommander.IParameterValidator;
 import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
 import com.sparrowwallet.drongo.Network;
 import org.slf4j.event.Level;
 
@@ -35,9 +37,21 @@ public class Args {
     @Parameter(names = { "--start", "-b" }, description = "Scan start block height or timestamp")
     public Long start;
 
+    @Parameter(names = {"--labels", "-a"}, description = "List of positive integers representing labels to scan for (change is always included)", validateWith = PositiveIntegerValidator.class)
+    public List<Integer> labels = new ArrayList<>();
+
     @Parameter(names = { "--follow", "-f" }, description = "Keep client open after initial scan to receive additional transaction")
     public boolean follow;
 
     @Parameter(names = { "--quiet", "-q" }, description = "Disable printing of the progress bar")
     public boolean quiet;
+
+    public static class PositiveIntegerValidator implements IParameterValidator {
+        public void validate(String name, String value) throws ParameterException {
+            int n = Integer.parseInt(value);
+            if(n < 0) {
+                throw new ParameterException("Parameter " + name + " should be positive (found " + value + ")");
+            }
+        }
+    }
 }

src/main/java/com/sparrowwallet/frigate/cli/ElectrumClientService.java

@@ -6,13 +6,16 @@ import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcMethod;
 import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcOptional;
 import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam;
 import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService;
-import com.sparrowwallet.frigate.index.TxEntry;
+import com.sparrowwallet.frigate.electrum.SilentPaymentsSubscription;
 
-import java.util.Collection;
+import java.util.List;
 
 @JsonRpcService
 @JsonRpcParams(ParamsType.ARRAY)
 public interface ElectrumClientService {
+    @JsonRpcMethod("server.version")
+    List<String> getServerVersion(@JsonRpcParam("client_name") String clientName, @JsonRpcParam("protocol_version") Object protocolVersion);
+
     @JsonRpcMethod("blockchain.silentpayments.subscribe")
-    String subscribeSilentPayments(@JsonRpcParam("scan_private_key") String scanPrivateKey, @JsonRpcParam("spend_public_key") String spendPublicKey, @JsonRpcParam("start") @JsonRpcOptional Long start);
+    SilentPaymentsSubscription subscribeSilentPayments(@JsonRpcParam("scan_private_key") String scanPrivateKey, @JsonRpcParam("spend_public_key") String spendPublicKey, @JsonRpcParam("start") @JsonRpcOptional Long start, @JsonRpcParam("labels") @JsonRpcOptional Integer[] labels);
 }

src/main/java/com/sparrowwallet/frigate/cli/ElectrumTransport.java

@@ -1,257 +0,0 @@
-package com.sparrowwallet.frigate.cli;
-
-import com.github.arteam.simplejsonrpc.client.Transport;
-import com.github.arteam.simplejsonrpc.server.JsonRpcServer;
-import com.google.common.net.HostAndPort;
-import com.google.gson.Gson;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.SocketFactory;
-import java.io.*;
-import java.net.*;
-import java.nio.charset.StandardCharsets;
-import java.util.Objects;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static com.sparrowwallet.frigate.electrum.ElectrumServerRunnable.DEFAULT_PORT;
-
-public class ElectrumTransport implements Transport, Closeable {
-    private static final Logger log = LoggerFactory.getLogger(ElectrumTransport.class);
-
-    private final HostAndPort electrumServer;
-    private Socket socket;
-    private String response;
-
-    private boolean firstRead = true;
-
-    private final CountDownLatch readReadySignal = new CountDownLatch(1);
-
-    private final ReentrantLock readLock = new ReentrantLock();
-    private final Condition readingCondition = readLock.newCondition();
-
-    private final ReentrantLock clientRequestLock = new ReentrantLock();
-    private boolean running = false;
-    private volatile boolean reading = true;
-    private boolean closed = false;
-    private Exception lastException;
-
-    private final Gson gson = new Gson();
-
-    private final JsonRpcServer jsonRpcServer = new JsonRpcServer();
-    private final SubscriptionService subscriptionService = new SubscriptionService();
-
-    public ElectrumTransport(HostAndPort electrumServer) {
-        this.electrumServer = electrumServer;
-        try {
-            String host = electrumServer.getHost();
-            int port = electrumServer.hasPort() ? electrumServer.getPort() : DEFAULT_PORT;
-
-            SocketFactory socketFactory = SocketFactory.getDefault();
-            this.socket = socketFactory.createSocket(host, port);
-            this.running = true;
-        } catch(UnknownHostException e) {
-            log.error("Unknown host: " + electrumServer.getHost());
-        } catch(IOException e) {
-            log.error("Error connecting to Electrum server: " + electrumServer.getHost());
-        }
-    }
-
-    @Override
-    public String pass(String request) throws IOException {
-        clientRequestLock.lock();
-        try {
-            Rpc sentRpc = request.startsWith("{") ? gson.fromJson(request, Rpc.class) : null;
-            Rpc recvRpc;
-            String recv;
-
-            writeRequest(request);
-            do {
-                recv = readResponse();
-                recvRpc = recv.startsWith("{") ? gson.fromJson(response, Rpc.class) : null;
-            } while(!Objects.equals(recvRpc, sentRpc));
-
-            return recv;
-        } finally {
-            clientRequestLock.unlock();
-        }
-    }
-
-    protected void writeRequest(String request) throws IOException {
-        log.debug("> " + request);
-
-        if(socket == null) {
-            throw new IllegalStateException("Socket connection has not been established.");
-        }
-
-        PrintWriter out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8)));
-        out.println(request);
-        out.flush();
-    }
-
-    private String readResponse() throws IOException {
-        if(firstRead) {
-            try {
-                //Ensure read thread has started
-                if(!readReadySignal.await(2, TimeUnit.SECONDS)) {
-                    throw new IOException("Read thread did not start");
-                }
-            } catch(InterruptedException e) {
-                throw new IOException("Read ready await interrupted");
-            }
-        }
-
-        try {
-            if(!readLock.tryLock(1, TimeUnit.SECONDS)) {
-                throw new IOException("No response from server");
-            }
-        } catch(InterruptedException e) {
-            throw new IOException("Read thread interrupted");
-        }
-
-        try {
-            if(firstRead) {
-                readingCondition.signal();
-                firstRead = false;
-            }
-
-            while(reading) {
-                try {
-                    readingCondition.await();
-                } catch(InterruptedException e) {
-                    //Restore interrupt status and break
-                    Thread.currentThread().interrupt();
-                    break;
-                }
-            }
-
-            if(lastException != null) {
-                throw new IOException("Error reading response: " + lastException.getMessage(), lastException);
-            }
-
-            reading = true;
-
-            readingCondition.signal();
-            return response;
-        } finally {
-            readLock.unlock();
-        }
-    }
-
-    public void readInputLoop() throws Exception {
-        readLock.lock();
-        readReadySignal.countDown();
-
-        try {
-            try {
-                //Don't start reading until first RPC request is sent
-                readingCondition.await();
-            } catch(InterruptedException e) {
-                Thread.currentThread().interrupt();
-            }
-
-            BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8));
-
-            while(running) {
-                try {
-                    String received = readInputStream(in);
-                    if(received.contains("method") && !received.contains("error")) {
-                        //Handle subscription notification
-                        jsonRpcServer.handle(received, subscriptionService);
-                    } else {
-                        //Handle client's response
-                        response = received;
-                        reading = false;
-                        readingCondition.signal();
-                        readingCondition.await();
-                    }
-                } catch(InterruptedException e) {
-                    //Restore interrupt status and continue
-                    Thread.currentThread().interrupt();
-                } catch(Exception e) {
-                    log.trace("Connection error while reading", e);
-                    if(running) {
-                        lastException = e;
-                        reading = false;
-                        readingCondition.signal();
-                        //Allow this thread to terminate as we will need to reconnect with a new transport anyway
-                        running = false;
-                    }
-                }
-            }
-        } catch(IOException e) {
-            if(!closed) {
-                log.error("Error opening socket inputstream", e);
-            }
-            if(running) {
-                lastException = e;
-                reading = false;
-                readingCondition.signal();
-                //Allow this thread to terminate as we will need to reconnect with a new transport anyway
-                running = false;
-            }
-        } finally {
-            readLock.unlock();
-        }
-    }
-
-    protected String readInputStream(BufferedReader in) throws IOException {
-        String response = readLine(in);
-
-        if(response == null) {
-            throw new IOException("Could not connect to server " + electrumServer);
-        }
-
-        log.debug("< " + response);
-
-        return response;
-    }
-
-    private String readLine(BufferedReader in) throws IOException {
-        while(!socket.isClosed()) {
-            try {
-                return in.readLine();
-            } catch(SocketTimeoutException e) {
-                //ignore and continue
-            }
-        }
-
-        return null;
-    }
-
-    public Exception getLastException() {
-        return lastException;
-    }
-
-    @Override
-    public void close() throws IOException {
-        if(socket != null) {
-            socket.close();
-        }
-        closed = true;
-    }
-
-    public static class Rpc {
-        public String id;
-
-        @Override
-        public boolean equals(Object o) {
-            if(this == o) {
-                return true;
-            }
-            if(o == null || getClass() != o.getClass()) {
-                return false;
-            }
-            Rpc rpc = (Rpc) o;
-            return Objects.equals(id, rpc.id);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(id);
-        }
-    }
-}

src/main/java/com/sparrowwallet/frigate/cli/FrigateCli.java

@@ -7,6 +7,10 @@ import com.google.common.net.HostAndPort;
 import com.sparrowwallet.drongo.Drongo;
 import com.sparrowwallet.drongo.Network;
 import com.sparrowwallet.frigate.Frigate;
+import com.sparrowwallet.frigate.electrum.ElectrumServerService;
+import com.sparrowwallet.frigate.electrum.ElectrumTransport;
+import com.sparrowwallet.frigate.electrum.SilentPaymentsSubscription;
+import com.sparrowwallet.frigate.io.Protocol;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -23,17 +27,19 @@ public class FrigateCli implements Thread.UncaughtExceptionHandler {
     private String scanPrivateKey;
     private String spendPublicKey;
     private Long start;
+    private final Integer[] labels;
 
     private static ElectrumTransport transport;
     private Thread reader;
 
     private static final EventBus EVENT_BUS = new EventBus();
 
-    public FrigateCli(HostAndPort server, String scanPrivateKey, String spendPublicKey, Long start) {
+    public FrigateCli(HostAndPort server, String scanPrivateKey, String spendPublicKey, Long start, Integer[] labels) {
         this.server = server;
         this.scanPrivateKey = scanPrivateKey;
         this.spendPublicKey = spendPublicKey;
         this.start = start;
+        this.labels = labels;
     }
 
     public void promptForMissingValues() {
@@ -70,9 +76,9 @@ public class FrigateCli implements Thread.UncaughtExceptionHandler {
     }
 
     public void connect() {
-        transport = new ElectrumTransport(server);
-        reader = new Thread(new ReadRunnable(), "ElectrumServerReadThread");
-        reader.setDaemon(true);
+        transport = new ElectrumTransport(server, Protocol.TCP, new SubscriptionService());
+        transport.connect();
+        reader = Thread.ofVirtual().name("ElectrumServerReadThread").unstarted(new ReadRunnable());
         reader.setUncaughtExceptionHandler(FrigateCli.this);
         reader.start();
     }
@@ -80,13 +86,14 @@ public class FrigateCli implements Thread.UncaughtExceptionHandler {
     public void scan(boolean follow, boolean quiet) {
         JsonRpcClient jsonRpcClient = new JsonRpcClient(getTransport());
         ElectrumClientService electrumClientService = jsonRpcClient.onDemand(ElectrumClientService.class);
-        String address = electrumClientService.subscribeSilentPayments(scanPrivateKey, spendPublicKey, start);
+        electrumClientService.getServerVersion(APP_NAME, ElectrumServerService.MIN_VERSION.get());
+        SilentPaymentsSubscription subscription = electrumClientService.subscribeSilentPayments(scanPrivateKey, spendPublicKey, start, labels);
 
         try {
-            ScanProgress scanProgress = new ScanProgress(address, !follow, !quiet);
+            ScanProgress scanProgress = new ScanProgress(subscription.address(), !follow, !quiet);
             getEventBus().register(scanProgress);
             if(!quiet) {
-                System.out.println("Scanning address " + address + "...");
+                System.out.println("Scanning address " + subscription.address() + "...");
             }
             scanProgress.waitForCompletion();
             getEventBus().unregister(scanProgress);
@@ -149,7 +156,12 @@ public class FrigateCli implements Thread.UncaughtExceptionHandler {
 
         HostAndPort server = HostAndPort.fromString(args.host == null ? "127.0.0.1" : args.host);
 
-        FrigateCli frigateCli = new FrigateCli(server, args.scanPrivateKey, args.spendPublicKey, args.start);
+        Integer[] labels = null;
+        if(args.labels != null) {
+            labels = args.labels.toArray(new Integer[0]);
+        }
+
+        FrigateCli frigateCli = new FrigateCli(server, args.scanPrivateKey, args.spendPublicKey, args.start, labels);
         frigateCli.promptForMissingValues();
         frigateCli.connect();
         frigateCli.scan(args.follow, args.quiet);

src/main/java/com/sparrowwallet/frigate/cli/ScanProgress.java

@@ -4,7 +4,7 @@ import com.google.common.eventbus.Subscribe;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.sparrowwallet.frigate.electrum.SilentPaymentsNotification;
-import com.sparrowwallet.frigate.index.TxEntry;
+import com.sparrowwallet.frigate.electrum.SilentPaymentsTxEntry;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -16,7 +16,7 @@ public class ScanProgress {
     private final String address;
     private final boolean canComplete;
     private final boolean showProgress;
-    private final List<TxEntry> results = new ArrayList<>();
+    private final List<SilentPaymentsTxEntry> results = new ArrayList<>();
 
     private volatile boolean isComplete = false;
     private volatile boolean isInitialComplete = false;
@@ -74,7 +74,7 @@ public class ScanProgress {
 
     public void waitForCompletion() throws InterruptedException {
         synchronized (completionLock) {
-            while (!isComplete) {
+            while(!isComplete) {
                 completionLock.wait();
             }
         }

src/main/java/com/sparrowwallet/frigate/cli/SubscriptionService.java

@@ -5,7 +5,7 @@ import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam;
 import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService;
 import com.sparrowwallet.frigate.electrum.SilentPaymentsNotification;
 import com.sparrowwallet.frigate.electrum.SilentPaymentsSubscription;
-import com.sparrowwallet.frigate.index.TxEntry;
+import com.sparrowwallet.frigate.electrum.SilentPaymentsTxEntry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -16,7 +16,7 @@ public class SubscriptionService {
     private static final Logger log = LoggerFactory.getLogger(SubscriptionService.class);
 
     @JsonRpcMethod("blockchain.silentpayments.subscribe")
-    public void silentPaymentsUpdate(@JsonRpcParam("subscription") SilentPaymentsSubscription subscription, @JsonRpcParam("progress") double progress, @JsonRpcParam("history") List<TxEntry> history) {
+    public void silentPaymentsUpdate(@JsonRpcParam("subscription") SilentPaymentsSubscription subscription, @JsonRpcParam("progress") double progress, @JsonRpcParam("history") List<SilentPaymentsTxEntry> history) {
         FrigateCli.getEventBus().post(new SilentPaymentsNotification(subscription, progress, history, null));
     }
 }

src/main/java/com/sparrowwallet/frigate/control/TrayManager.java

@@ -0,0 +1,111 @@
+package com.sparrowwallet.frigate.control;
+
+import com.google.common.eventbus.Subscribe;
+import com.sparrowwallet.drongo.Network;
+import com.sparrowwallet.drongo.OsType;
+import com.sparrowwallet.frigate.Frigate;
+import com.sparrowwallet.frigate.electrum.ElectrumBlockHeader;
+import com.sparrowwallet.frigate.index.SilentPaymentsBlocksIndexUpdate;
+import com.sparrowwallet.frigate.io.Config;
+import com.sparrowwallet.frigate.io.Server;
+import com.sparrowwallet.frigate.io.Storage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BaseMultiResolutionImage;
+import java.io.IOException;
+import java.text.NumberFormat;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TrayManager {
+    private static final Logger log = LoggerFactory.getLogger(TrayManager.class);
+
+    private final TrayIcon trayIcon;
+    private final PopupMenu popupMenu = new PopupMenu();
+    private final MenuItem statusItem;
+
+    private int tipHeight = -1;
+    private int indexedHeight = -1;
+
+    public TrayManager() {
+        if(!isSupported()) {
+            throw new UnsupportedOperationException("System tray is not supported on this platform.");
+        }
+
+        SystemTray tray = SystemTray.getSystemTray();
+
+        try {
+            List<Image> imgList = new ArrayList<>();
+            imgList.add(ImageIO.read(getClass().getResource("/image/frigate-white-small.png")));
+            imgList.add(ImageIO.read(getClass().getResource("/image/frigate-white-small@2x.png")));
+            imgList.add(ImageIO.read(getClass().getResource("/image/frigate-white-small@3x.png")));
+
+            BaseMultiResolutionImage mrImage = new BaseMultiResolutionImage(imgList.toArray(new Image[0]));
+            this.trayIcon = new TrayIcon(mrImage, "Frigate", popupMenu);
+
+            MenuItem versionItem = new MenuItem(Frigate.SERVER_NAME + " " + Frigate.SERVER_VERSION + " (" + Network.get().getName() + ")");
+            versionItem.setEnabled(false);
+            popupMenu.add(versionItem);
+
+            statusItem = new MenuItem(Config.get().getCore().shouldConnect() ? "Starting..." : "Indexing Disabled");
+            statusItem.setEnabled(false);
+            popupMenu.add(statusItem);
+
+            MenuItem configItem = new MenuItem("Open Config Folder");
+            configItem.addActionListener(e -> {
+                try {
+                    Desktop.getDesktop().open(Storage.getFrigateDir());
+                } catch(IOException ex) {
+                    log.error("Could not open config folder", ex);
+                }
+            });
+            popupMenu.add(configItem);
+
+            MenuItem quitItem = new MenuItem("Quit Frigate");
+            quitItem.addActionListener(e -> System.exit(0));
+            popupMenu.add(quitItem);
+
+            tray.add(trayIcon);
+        } catch(IOException | AWTException e) {
+            log.error("Could not initialize system tray", e);
+            throw new IllegalStateException(e);
+        }
+    }
+
+    @Subscribe
+    public void electrumBlockHeader(ElectrumBlockHeader header) {
+        tipHeight = header.height();
+        indexedHeight = -1;
+        updateStatusItem();
+    }
+
+    @Subscribe
+    public void silentPaymentsBlocksIndexUpdate(SilentPaymentsBlocksIndexUpdate update) {
+        indexedHeight = update.toBlockHeight();
+        updateStatusItem();
+    }
+
+    private void updateStatusItem() {
+        if(tipHeight == -1) {
+            statusItem.setLabel("Starting...");
+        } else if(indexedHeight >= 0 && indexedHeight < tipHeight) {
+            NumberFormat nf = NumberFormat.getIntegerInstance();
+            statusItem.setLabel("Indexing: block " + nf.format(indexedHeight) + " / " + nf.format(tipHeight));
+        } else {
+            Config.ServerConfig sc = Config.get().getServer();
+            Server tcpServer = sc.getTcpServer();
+            Server sslServer = sc.getSslServer();
+            List<String> ports = new ArrayList<>(2);
+            if(tcpServer != null) ports.add(Integer.toString(tcpServer.getHostAndPort().getPort()));
+            if(sslServer != null) ports.add(Integer.toString(sslServer.getHostAndPort().getPort()));
+            statusItem.setLabel("Electrum server on port " + String.join("/", ports));
+        }
+    }
+
+    public static boolean isSupported() {
+        return OsType.getCurrent() == OsType.MACOS && Desktop.isDesktopSupported() && SystemTray.isSupported();
+    }
+}

src/main/java/com/sparrowwallet/frigate/electrum/BackendSubscriptionService.java

@@ -0,0 +1,15 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcMethod;
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcOptional;
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam;
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService;
+import com.sparrowwallet.frigate.Frigate;
+
+@JsonRpcService
+public class BackendSubscriptionService {
+    @JsonRpcMethod("blockchain.scripthash.subscribe")
+    public void scriptHashStatusUpdated(@JsonRpcParam("scripthash") final String scriptHash, @JsonRpcOptional @JsonRpcParam("status") final String status) {
+        Frigate.getEventBus().post(new ScriptHashStatus(scriptHash, status));
+    }
+}

src/main/java/com/sparrowwallet/frigate/electrum/ElectrumBackendService.java

@@ -0,0 +1,66 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.github.arteam.simplejsonrpc.client.JsonRpcParams;
+import com.github.arteam.simplejsonrpc.client.ParamsType;
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcMethod;
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcOptional;
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam;
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService;
+
+import java.util.Collection;
+import java.util.List;
+
+@JsonRpcService
+@JsonRpcParams(ParamsType.ARRAY)
+public interface ElectrumBackendService {
+    @JsonRpcMethod("server.version")
+    List<String> getServerVersion(@JsonRpcParam("client_name") String clientName, @JsonRpcParam("protocol_version") Object protocolVersion);
+
+    @JsonRpcMethod("server.features")
+    ServerFeatures getServerFeatures();
+
+    @JsonRpcMethod("server.add_peer")
+    boolean addPeer(@JsonRpcParam("features") ServerFeatures features);
+
+    @JsonRpcMethod("server.donation_address")
+    String getDonationAddress();
+
+    @JsonRpcMethod("server.peers.subscribe")
+    List<ServerPeer> subscribePeers();
+
+    @JsonRpcMethod("mempool.get_fee_histogram")
+    List<List<Number>> getFeeHistogram();
+
+    @JsonRpcMethod("blockchain.scripthash.subscribe")
+    String subscribeScriptHash(@JsonRpcParam("scripthash") String scriptHash);
+
+    @JsonRpcMethod("blockchain.scripthash.unsubscribe")
+    String unsubscribeScriptHash(@JsonRpcParam("scripthash") String scriptHash);
+
+    @JsonRpcMethod("blockchain.scripthash.get_balance")
+    ScriptHashBalance getBalance(@JsonRpcParam("scripthash") String scriptHash);
+
+    @JsonRpcMethod("blockchain.scripthash.get_history")
+    Collection<TxEntry> getHistory(@JsonRpcParam("scripthash") String scriptHash);
+
+    @JsonRpcMethod("blockchain.scripthash.get_mempool")
+    Collection<TxEntry> getMempool(@JsonRpcParam("scripthash") String scriptHash);
+
+    @JsonRpcMethod("blockchain.scripthash.listunspent")
+    Collection<UnspentOutput> listUnspent(@JsonRpcParam("scripthash") String scriptHash);
+
+    @JsonRpcMethod("blockchain.block.header")
+    Object getBlockHeader(@JsonRpcParam("height") int height, @JsonRpcParam("cp_height") Integer cpHeight);
+
+    @JsonRpcMethod("blockchain.block.headers")
+    Object getBlockHeaders(@JsonRpcParam("start_height") int startHeight, @JsonRpcParam("count") int count);
+
+    @JsonRpcMethod("blockchain.block.headers")
+    Object getBlockHeaders(@JsonRpcParam("start_height") int startHeight, @JsonRpcParam("count") int count, @JsonRpcParam("cp_height") Integer cpHeight);
+
+    @JsonRpcMethod("blockchain.transaction.get_merkle")
+    TransactionMerkle getTransactionMerkle(@JsonRpcParam("tx_hash") String txHash, @JsonRpcParam("height") int height);
+
+    @JsonRpcMethod("blockchain.transaction.id_from_pos")
+    Object getTransactionIdFromPos(@JsonRpcParam("height") int height, @JsonRpcParam("tx_pos") int txPos, @JsonRpcParam("merkle") @JsonRpcOptional Boolean merkle);
+}

src/main/java/com/sparrowwallet/frigate/electrum/ElectrumNotificationService.java

@@ -4,7 +4,6 @@ import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcMethod;
 import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcOptional;
 import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcParam;
 import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcService;
-import com.sparrowwallet.frigate.index.TxEntry;
 
 import java.util.List;
 
@@ -17,5 +16,5 @@ public interface ElectrumNotificationService {
     void notifyScriptHash(@JsonRpcParam("scripthash") String scriptHash, @JsonRpcOptional @JsonRpcParam("status") String status);
 
     @JsonRpcMethod("blockchain.silentpayments.subscribe")
-    void notifySilentPayments(@JsonRpcParam("subscription") SilentPaymentsSubscription silentPaymentsSubscription, @JsonRpcParam("progress") double progress, @JsonRpcParam("history") List<TxEntry> history);
+    void notifySilentPayments(@JsonRpcParam("subscription") SilentPaymentsSubscription silentPaymentsSubscription, @JsonRpcParam("progress") double progress, @JsonRpcParam("history") List<SilentPaymentsTxEntry> history);
 }

src/main/java/com/sparrowwallet/frigate/electrum/ElectrumNotificationTransport.java

@@ -3,24 +3,17 @@ package com.sparrowwallet.frigate.electrum;
 import com.github.arteam.simplejsonrpc.client.Transport;
 
 import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.net.Socket;
-import java.nio.charset.StandardCharsets;
 
 public class ElectrumNotificationTransport implements Transport {
-    private final Socket clientSocket;
+    private final RequestHandler requestHandler;
 
-    public ElectrumNotificationTransport(Socket clientSocket) {
-        this.clientSocket = clientSocket;
+    public ElectrumNotificationTransport(RequestHandler requestHandler) {
+        this.requestHandler = requestHandler;
     }
 
     @Override
     public String pass(String request) throws IOException {
-        PrintWriter out = new PrintWriter(new OutputStreamWriter(clientSocket.getOutputStream(), StandardCharsets.UTF_8));
-        out.println(request);
-        out.flush();
-
+        requestHandler.writeLine(request);
         return "{\"result\":{},\"error\":null,\"id\":1}";
     }
 }

src/main/java/com/sparrowwallet/frigate/electrum/ElectrumServerRunnable.java

@@ -1,87 +1,162 @@
 package com.sparrowwallet.frigate.electrum;
 
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.sparrowwallet.frigate.ConfigurationException;
 import com.sparrowwallet.frigate.bitcoind.BitcoindClient;
 import com.sparrowwallet.frigate.index.IndexQuerier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLServerSocket;
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadFactory;
 
 public class ElectrumServerRunnable implements Runnable {
     private static final Logger log = LoggerFactory.getLogger(ElectrumServerRunnable.class);
-    public static final int DEFAULT_PORT = 57001;
+    private static final int LISTEN_BACKLOG = 50;
+
+    private static final Set<String> ALLOWED_TLS_PROTOCOLS = Set.of("TLSv1.2", "TLSv1.3");
 
     private final BitcoindClient bitcoindClient;
     private final IndexQuerier indexQuerier;
+    private final InetSocketAddress tcpBind;
+    private final InetSocketAddress sslBind;
+    private final List<ServerSocket> serverSockets = new ArrayList<>();
 
-    protected ServerSocket serverSocket = null;
-    protected boolean stopped = false;
-    protected Thread runningThread = null;
-    protected ExecutorService requestPool = Executors.newFixedThreadPool(10, r -> {
-        ThreadFactory namedThreadFactory = new ThreadFactoryBuilder().setNameFormat("ElectrumServerRequest-%d").build();
-        Thread t = namedThreadFactory.newThread(r);
-        t.setDaemon(true);
-        return t;
-    });
+    protected volatile boolean stopped = false;
+    protected ExecutorService requestPool = Executors.newThreadPerTaskExecutor(Thread.ofVirtual().name("ElectrumServerRequest-", 0).factory());
 
-    public ElectrumServerRunnable(BitcoindClient bitcoindClient, IndexQuerier indexQuerier) {
+    public ElectrumServerRunnable(BitcoindClient bitcoindClient, IndexQuerier indexQuerier, InetSocketAddress tcpBind, InetSocketAddress sslBind, SSLContext sslContext) {
         this.bitcoindClient = bitcoindClient;
         this.indexQuerier = indexQuerier;
-        openServerSocket();
+        this.tcpBind = tcpBind;
+        this.sslBind = sslBind;
+
+        if(tcpBind == null && sslBind == null) {
+            throw new ConfigurationException("At least one of tcp or ssl must be enabled under [server] in config.toml");
+        }
+
+        if(sslBind != null && sslContext == null) {
+            throw new ConfigurationException("SSL: ssl listener configured but no SSLContext was supplied");
+        }
+
+        openServerSockets(sslContext);
     }
 
-    public int getPort() {
-        return serverSocket.getLocalPort();
+    public InetSocketAddress getTcpBind() {
+        return tcpBind;
+    }
+
+    public InetSocketAddress getSslBind() {
+        return sslBind;
     }
 
     public void run() {
-        synchronized(this) {
-            this.runningThread = Thread.currentThread();
+        StringBuilder banner = new StringBuilder("Electrum server listening on");
+        if(tcpBind != null) banner.append(" tcp://").append(formatBind(tcpBind));
+        if(sslBind != null) banner.append(" ssl://").append(formatBind(sslBind));
+        log.info(banner.toString());
+
+        CountDownLatch done = new CountDownLatch(serverSockets.size());
+        for(ServerSocket ss : serverSockets) {
+            Thread.ofVirtual().name("ElectrumAccept-" + ss.getLocalPort()).start(() -> {
+                try {
+                    acceptLoop(ss);
+                } finally {
+                    done.countDown();
+                }
+            });
         }
 
-        log.info("Electrum server listening on port {}", getPort());
-
-        while(!isStopped()) {
-            Socket clientSocket;
-            try {
-                clientSocket = this.serverSocket.accept();
-            } catch(IOException e) {
-                if(isStopped()) {
-                    break;
-                }
-                throw new RuntimeException("Error accepting client connection", e);
-            }
-            RequestHandler requestHandler = new RequestHandler(clientSocket, bitcoindClient, indexQuerier);
-            this.requestPool.execute(requestHandler);
+        try {
+            done.await();
+        } catch(InterruptedException e) {
+            Thread.currentThread().interrupt();
         }
 
         this.requestPool.shutdown();
     }
 
-    private synchronized boolean isStopped() {
-        return stopped;
+    private void acceptLoop(ServerSocket serverSocket) {
+        while(!stopped) {
+            Socket clientSocket;
+            try {
+                clientSocket = serverSocket.accept();
+            } catch(IOException e) {
+                if(stopped) {
+                    return;
+                }
+                log.error("Error accepting client connection on port " + serverSocket.getLocalPort(), e);
+                return;
+            }
+            RequestHandler requestHandler = new RequestHandler(clientSocket, bitcoindClient, indexQuerier);
+            this.requestPool.execute(requestHandler);
+        }
     }
 
     public synchronized void stop() {
         stopped = true;
-        try {
-            serverSocket.close();
-        } catch(IOException e) {
-            throw new RuntimeException("Error closing server", e);
+        for(ServerSocket ss : serverSockets) {
+            try {
+                ss.close();
+            } catch(IOException e) {
+                log.error("Error closing server socket on port " + ss.getLocalPort(), e);
+            }
         }
     }
 
-    private void openServerSocket() {
+    private void openServerSockets(SSLContext sslContext) {
         try {
-            serverSocket = new ServerSocket(DEFAULT_PORT);
+            if(tcpBind != null) {
+                ServerSocket plain = new ServerSocket(tcpBind.getPort(), LISTEN_BACKLOG, tcpBind.getAddress());
+                serverSockets.add(plain);
+            }
+            if(sslBind != null) {
+                SSLServerSocket sslSocket = (SSLServerSocket)sslContext.getServerSocketFactory().createServerSocket(sslBind.getPort(), LISTEN_BACKLOG, sslBind.getAddress());
+                sslSocket.setNeedClientAuth(false);
+                sslSocket.setEnabledProtocols(restrictedProtocols(sslSocket.getSupportedProtocols()));
+                serverSockets.add(sslSocket);
+            }
         } catch(IOException e) {
+            for(ServerSocket opened : serverSockets) {
+                try {
+                    opened.close();
+                } catch(IOException ignored) {
+                    //ignore
+                }
+            }
+            serverSockets.clear();
             throw new RuntimeException("Cannot open electrum server port", e);
         }
     }
+
+    private static String formatBind(InetSocketAddress addr) {
+        return addr.getAddress().getHostAddress() + ":" + addr.getPort();
+    }
+
+    private static String[] restrictedProtocols(String[] supported) {
+        List<String> enabled = new ArrayList<>(2);
+        Set<String> supportedSet = new HashSet<>(Arrays.asList(supported));
+        for(String p : ALLOWED_TLS_PROTOCOLS) {
+            if(supportedSet.contains(p)) {
+                enabled.add(p);
+            }
+        }
+
+        if(enabled.isEmpty()) {
+            throw new ConfigurationException("SSL: JVM supports neither TLSv1.2 nor TLSv1.3 (supported: " + Arrays.toString(supported) + ")");
+        }
+
+        return enabled.toArray(new String[0]);
+    }
 }

src/main/java/com/sparrowwallet/frigate/electrum/ElectrumServerService.java

@@ -1,5 +1,6 @@
 package com.sparrowwallet.frigate.electrum;
 
+import com.github.arteam.simplejsonrpc.client.JsonRpcClient;
 import com.github.arteam.simplejsonrpc.client.exception.JsonRpcException;
 import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcMethod;
 import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcOptional;
@@ -11,12 +12,11 @@ import com.sparrowwallet.drongo.crypto.ECKey;
 import com.sparrowwallet.drongo.protocol.Transaction;
 import com.sparrowwallet.drongo.silentpayments.SilentPaymentScanAddress;
 import com.sparrowwallet.frigate.Frigate;
-import com.sparrowwallet.frigate.bitcoind.BitcoindClient;
-import com.sparrowwallet.frigate.bitcoind.BlockStats;
-import com.sparrowwallet.frigate.bitcoind.FeeInfo;
-import com.sparrowwallet.frigate.bitcoind.MempoolInfo;
+import com.sparrowwallet.frigate.bitcoind.*;
 import com.sparrowwallet.frigate.index.IndexQuerier;
-import com.sparrowwallet.frigate.index.TxEntry;
+import com.sparrowwallet.frigate.io.Config;
+import com.sparrowwallet.frigate.io.Protocol;
+import com.sparrowwallet.frigate.io.Server;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -26,45 +26,174 @@ import java.util.*;
 @JsonRpcService
 public class ElectrumServerService {
     private static final Logger log = LoggerFactory.getLogger(ElectrumServerService.class);
-    private static final Version VERSION = new Version("1.4");
-    private static final double DEFAULT_FEE_RATE = 0.00001d;
+    public static final Version MIN_VERSION = new Version("1.4");
+    public static final Version MAX_DEFAULT_VERSION = new Version("1.4.2");
+    public static final Version MAX_SUBMIT_PACKAGE_VERSION = new Version("1.6");
+    public static final List<Integer> SILENT_PAYMENTS_SUPPORTED_VERSIONS = List.of(0);
+    private static final int METHOD_NOT_FOUND = -32601;
 
     private final BitcoindClient bitcoindClient;
     private final RequestHandler requestHandler;
     private final IndexQuerier indexQuerier;
+    private final ElectrumBackendService electrumBackendService;
+    private Version protocolVersion;
+    private String genesisHash;
 
-    public ElectrumServerService(BitcoindClient bitcoindClient, RequestHandler requestHandler, IndexQuerier indexQuerier) {
+    public ElectrumServerService(BitcoindClient bitcoindClient, RequestHandler requestHandler, IndexQuerier indexQuerier, ElectrumTransport backendTransport) {
         this.bitcoindClient = bitcoindClient;
         this.requestHandler = requestHandler;
         this.indexQuerier = indexQuerier;
+
+        if(backendTransport != null) {
+            JsonRpcClient jsonRpcClient = new JsonRpcClient(backendTransport);
+            this.electrumBackendService = jsonRpcClient.onDemand(ElectrumBackendService.class);
+        } else {
+            electrumBackendService = null;
+        }
     }
 
     public IndexQuerier getIndexQuerier() {
         return indexQuerier;
     }
 
+    private void checkVersionNegotiated() {
+        if(protocolVersion == null) {
+            throw new VersionNotNegotiatedException();
+        }
+    }
+
+    private Version getMaxSupportedVersion() {
+        return bitcoindClient != null && bitcoindClient.containsSubmitPackage() ? MAX_SUBMIT_PACKAGE_VERSION : MAX_DEFAULT_VERSION;
+    }
+
     @JsonRpcMethod("server.version")
-    public List<String> getServerVersion(@JsonRpcParam("client_name") String clientName, @JsonRpcParam("protocol_version") String[] protocolVersion) throws UnsupportedVersionException {
-        String version = protocolVersion.length > 1 ? protocolVersion[1] : protocolVersion[0];
-        Version clientVersion = new Version(version);
-        if(clientVersion.compareTo(VERSION) < 0) {
-            throw new UnsupportedVersionException(version);
+    public List<String> getServerVersion(@JsonRpcParam("client_name") String clientName, @JsonRpcParam("protocol_version") Object protocolVersion) throws UnsupportedVersionException {
+        Version clientVersion = new Version(switch(protocolVersion) {
+            case String s -> s;
+            case List<?> versions -> {
+                if(versions.isEmpty()) throw new IllegalArgumentException("protocol_version list cannot be empty");
+                yield versions.size() > 1 ? versions.get(1).toString() : versions.get(0).toString();
+            }
+            case String[] versions -> {
+                if(versions.length == 0) throw new IllegalArgumentException("protocol_version array cannot be empty");
+                yield versions.length > 1 ? versions[1] : versions[0];
+            }
+            case null, default -> throw new IllegalArgumentException("Invalid protocol_version type: " + protocolVersion);
+        });
+
+        Version backendVersion = clientVersion;
+        if(electrumBackendService != null) {
+            List<String> backendVersions = electrumBackendService.getServerVersion(clientName, protocolVersion);
+            if(backendVersions != null && !backendVersions.isEmpty()) {
+                backendVersion = new Version(backendVersions.getLast());
+            }
         }
 
-        return List.of(Frigate.SERVER_NAME + " " + Frigate.SERVER_VERSION, VERSION.get());
+        Version version = backendVersion.compareTo(clientVersion) < 0 ? backendVersion : clientVersion;
+        if(version.compareTo(MIN_VERSION) < 0) {
+            throw new UnsupportedVersionException(version.get());
+        }
+
+        this.protocolVersion = version.compareTo(getMaxSupportedVersion()) > 0 ? getMaxSupportedVersion() : version;
+        return List.of(Frigate.SERVER_NAME + " " + Frigate.SERVER_VERSION, this.protocolVersion.get());
     }
 
     @JsonRpcMethod("server.banner")
     public String getServerBanner() {
-        return Frigate.SERVER_NAME + " " + Frigate.SERVER_VERSION + "\n" + bitcoindClient.getNetworkInfo().subversion() + (bitcoindClient.getNetworkInfo().networkactive() ? "" : " (disconnected)");
+        checkVersionNegotiated();
+        return Frigate.SERVER_NAME + " " + Frigate.SERVER_VERSION + (bitcoindClient != null ? "\n" + bitcoindClient.getNetworkInfo().subversion() + (bitcoindClient.getNetworkInfo().networkactive() ? "" : " (disconnected)") : "");
+    }
+
+    @JsonRpcMethod("server.features")
+    public ServerFeatures getServerFeatures() {
+        checkVersionNegotiated();
+        Map<String, ServerFeatures.HostInfo> ourHosts = buildAdvertisedHosts(Config.get().getServer().getAdvertisedHosts());
+
+        if(electrumBackendService != null) {
+            try {
+                return electrumBackendService.getServerFeatures().withHosts(ourHosts).withSilentPayments(SILENT_PAYMENTS_SUPPORTED_VERSIONS);
+            } catch(JsonRpcException e) {
+                if(e.getErrorMessage() == null || e.getErrorMessage().getCode() != METHOD_NOT_FOUND) {
+                    throw e;
+                }
+                log.debug("Backend does not support server.features, returning local response");
+            }
+        }
+
+        return new ServerFeatures(ourHosts, getGenesisHash(), "sha256", Frigate.SERVER_NAME + " " + Frigate.SERVER_VERSION,
+                protocolVersion.get(), MIN_VERSION.get(), null, SILENT_PAYMENTS_SUPPORTED_VERSIONS);
+    }
+
+    static Map<String, ServerFeatures.HostInfo> buildAdvertisedHosts(List<Server> servers) {
+        Map<String, ServerFeatures.HostInfo> result = new LinkedHashMap<>();
+        for(Server server : servers) {
+            int port = server.getHostAndPort().getPort();
+            ServerFeatures.HostInfo current = result.get(server.getHost());
+            Integer tcp = current == null ? null : current.tcp_port();
+            Integer ssl = current == null ? null : current.ssl_port();
+            if(server.getProtocol() == Protocol.TCP) {
+                tcp = port;
+            } else if(server.getProtocol() == Protocol.SSL) {
+                ssl = port;
+            }
+            result.put(server.getHost(), new ServerFeatures.HostInfo(tcp, ssl));
+        }
+        return result;
+    }
+
+    private String getGenesisHash() {
+        if(genesisHash == null && bitcoindClient != null) {
+            try {
+                genesisHash = bitcoindClient.getBitcoindService().getBlockHash(0);
+            } catch(Exception e) {
+                log.debug("Could not fetch genesis hash from bitcoind", e);
+            }
+        }
+
+        return genesisHash;
+    }
+
+    @JsonRpcMethod("server.add_peer")
+    public boolean addPeer(@JsonRpcParam("features") ServerFeatures features) {
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            return electrumBackendService.addPeer(features);
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use server.add_peer");
+    }
+
+    @JsonRpcMethod("server.donation_address")
+    public String getDonationAddress() {
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            return electrumBackendService.getDonationAddress();
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use server.donation_address");
+    }
+
+    @JsonRpcMethod("server.peers.subscribe")
+    public List<ServerPeer> subscribePeers() {
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            return electrumBackendService.subscribePeers();
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use server.peers.subscribe");
     }
 
     @JsonRpcMethod("blockchain.estimatefee")
-    public Double estimateFee(@JsonRpcParam("number") int blocks) throws BitcoindIOException {
+    public Double estimateFee(@JsonRpcParam("number") int blocks, @JsonRpcParam("mode") @JsonRpcOptional String mode) throws BitcoindIOException {
+        checkVersionNegotiated();
+        if(bitcoindClient == null) {
+            return MempoolInfo.DEFAULT_FEE_RATE;
+        }
+
         try {
-            FeeInfo feeInfo = bitcoindClient.getBitcoindService().estimateSmartFee(blocks);
+            FeeInfo feeInfo = bitcoindClient.getBitcoindService().estimateSmartFee(blocks, mode);
             if(feeInfo == null || feeInfo.feerate() == null) {
-                return DEFAULT_FEE_RATE;
+                return MempoolInfo.DEFAULT_FEE_RATE;
             }
 
             return feeInfo.feerate();
@@ -75,11 +204,21 @@ public class ElectrumServerService {
 
     @JsonRpcMethod("mempool.get_fee_histogram")
     public List<List<Number>> getFeeHistogram() {
-        throw new UnsupportedOperationException("Not yet implemented");
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            return electrumBackendService.getFeeHistogram();
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use mempool.get_fee_histogram");
     }
 
     @JsonRpcMethod("blockchain.relayfee")
     public Double getRelayFee() throws BitcoindIOException {
+        checkVersionNegotiated();
+        if(bitcoindClient == null) {
+            return MempoolInfo.DEFAULT_FEE_RATE;
+        }
+
         try {
             MempoolInfo mempoolInfo = bitcoindClient.getBitcoindService().getMempoolInfo();
             return mempoolInfo.minrelaytxfee();
@@ -88,16 +227,45 @@ public class ElectrumServerService {
         }
     }
 
+    @JsonRpcMethod("mempool.get_info")
+    public MempoolInfo getMempoolInfo() throws BitcoindIOException {
+        checkVersionNegotiated();
+        if(bitcoindClient == null) {
+            return MempoolInfo.DEFAULT;
+        }
+
+        try {
+            return bitcoindClient.getBitcoindService().getMempoolInfo();
+        } catch(IllegalStateException e) {
+            throw new BitcoindIOException(e);
+        }
+    }
+
     @JsonRpcMethod("blockchain.headers.subscribe")
     public ElectrumBlockHeader subscribeHeaders() {
-        requestHandler.setHeadersSubscribed(true);
-        return bitcoindClient.getTip();
+        checkVersionNegotiated();
+        if(bitcoindClient == null) {
+            throw new UnsupportedOperationException("Configure coreServer to use blockchain.headers.subscribe");
+        }
+        ElectrumBlockHeader tip = bitcoindClient.getTip();
+        requestHandler.runAfterResponse(() -> {
+            requestHandler.setHeadersSubscribed(true);
+            ElectrumBlockHeader currentTip = bitcoindClient.getTip();
+            if(currentTip != null && !currentTip.equals(tip)) {
+                requestHandler.notifyHeaders(currentTip);
+            }
+        });
+        return tip;
     }
 
     @JsonRpcMethod("server.ping")
-    public void ping() throws BitcoindIOException {
+    public Object ping() throws BitcoindIOException {
+        checkVersionNegotiated();
         try {
-            bitcoindClient.getBitcoindService().uptime();
+            if(bitcoindClient != null) {
+                bitcoindClient.getBitcoindService().uptime();
+            }
+            return null;
         } catch(IllegalStateException e) {
             throw new BitcoindIOException(e);
         }
@@ -105,21 +273,85 @@ public class ElectrumServerService {
 
     @JsonRpcMethod("blockchain.scripthash.subscribe")
     public String subscribeScriptHash(@JsonRpcParam("scripthash") String scriptHash) {
-        throw new UnsupportedOperationException("Not yet implemented");
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            requestHandler.subscribeScriptHash(scriptHash);
+            try {
+                return electrumBackendService.subscribeScriptHash(scriptHash);
+            } catch(RuntimeException e) {
+                requestHandler.unsubscribeScriptHash(scriptHash);
+                throw e;
+            }
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use blockchain.scripthash.subscribe");
     }
 
     @JsonRpcMethod("blockchain.scripthash.unsubscribe")
     public String unsubscribeScriptHash(@JsonRpcParam("scripthash") String scriptHash) {
-        throw new UnsupportedOperationException("Not yet implemented");
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            requestHandler.unsubscribeScriptHash(scriptHash);
+            return electrumBackendService.unsubscribeScriptHash(scriptHash);
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use blockchain.scripthash.unsubscribe");
+    }
+
+    @JsonRpcMethod("blockchain.scripthash.get_balance")
+    public ScriptHashBalance getBalance(@JsonRpcParam("scripthash") String scriptHash) {
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            return electrumBackendService.getBalance(scriptHash);
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use blockchain.scripthash.get_balance");
     }
 
     @JsonRpcMethod("blockchain.scripthash.get_history")
     public Collection<TxEntry> getHistory(@JsonRpcParam("scripthash") String scriptHash) {
-        throw new UnsupportedOperationException("Not yet implemented");
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            return electrumBackendService.getHistory(scriptHash);
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use blockchain.scripthash.get_history");
+    }
+
+    @JsonRpcMethod("blockchain.scripthash.get_mempool")
+    public Collection<TxEntry> getMempool(@JsonRpcParam("scripthash") String scriptHash) {
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            return electrumBackendService.getMempool(scriptHash);
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use blockchain.scripthash.get_mempool");
+    }
+
+    @JsonRpcMethod("blockchain.scripthash.listunspent")
+    public Collection<UnspentOutput> listUnspent(@JsonRpcParam("scripthash") String scriptHash) {
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            return electrumBackendService.listUnspent(scriptHash);
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use blockchain.scripthash.listunspent");
     }
 
     @JsonRpcMethod("blockchain.block.header")
-    public String getBlockHeader(@JsonRpcParam("height") int height) throws BitcoindIOException, BlockNotFoundException {
+    public Object getBlockHeader(@JsonRpcParam("height") int height, @JsonRpcParam("cp_height") @JsonRpcOptional Integer cpHeight) throws BitcoindIOException, BlockNotFoundException {
+        checkVersionNegotiated();
+        if(cpHeight != null && cpHeight > 0) {
+            if(electrumBackendService != null) {
+                return electrumBackendService.getBlockHeader(height, cpHeight);
+            }
+            throw new UnsupportedOperationException("Configure backendElectrumServer to use cp_height");
+        }
+
+        if(bitcoindClient == null) {
+            throw new UnsupportedOperationException("Configure coreServer to use blockchain.block.header");
+        }
+
         try {
             String blockHash = bitcoindClient.getBitcoindService().getBlockHash(height);
             return bitcoindClient.getBitcoindService().getBlockHeader(blockHash, false);
@@ -130,8 +362,27 @@ public class ElectrumServerService {
         }
     }
 
+    @JsonRpcMethod("blockchain.block.headers")
+    public Object getBlockHeaders(@JsonRpcParam("start_height") int startHeight, @JsonRpcParam("count") int count, @JsonRpcParam("cp_height") @JsonRpcOptional Integer cpHeight) {
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            if(cpHeight != null && cpHeight > 0) {
+                return electrumBackendService.getBlockHeaders(startHeight, count, cpHeight);
+            } else {
+                return electrumBackendService.getBlockHeaders(startHeight, count);
+            }
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use blockchain.block.headers");
+    }
+
     @JsonRpcMethod("blockchain.block.stats")
     public BlockStats getBlockStats(@JsonRpcParam("height") int height) throws BitcoindIOException, BlockNotFoundException {
+        checkVersionNegotiated();
+        if(bitcoindClient == null) {
+            throw new UnsupportedOperationException("Configure coreServer to use blockchain.block.stats");
+        }
+
         try {
             return bitcoindClient.getBitcoindService().getBlockStats(height);
         } catch(JsonRpcException e) {
@@ -144,6 +395,11 @@ public class ElectrumServerService {
     @JsonRpcMethod("blockchain.transaction.get")
     @SuppressWarnings("unchecked")
     public Object getTransaction(@JsonRpcParam("tx_hash") String tx_hash, @JsonRpcParam("verbose") @JsonRpcOptional boolean verbose) throws BitcoindIOException, TransactionNotFoundException {
+        checkVersionNegotiated();
+        if(bitcoindClient == null) {
+            throw new UnsupportedOperationException("Configure coreServer to use blockchain.transaction.get");
+        }
+
         if(verbose) {
             try {
                 return bitcoindClient.getBitcoindService().getRawTransaction(tx_hash, true);
@@ -178,7 +434,7 @@ public class ElectrumServerService {
                 } catch(JsonRpcException ex) {
                     throw new TransactionNotFoundException(ex.getErrorMessage());
                 } catch(IllegalStateException ex) {
-                    throw new BitcoindIOException(e);
+                    throw new BitcoindIOException(ex);
                 }
             } catch(IllegalStateException e) {
                 throw new BitcoindIOException(e);
@@ -186,8 +442,33 @@ public class ElectrumServerService {
         }
     }
 
+    @JsonRpcMethod("blockchain.transaction.get_merkle")
+    public TransactionMerkle getTransactionMerkle(@JsonRpcParam("tx_hash") String txHash, @JsonRpcParam("height") int height) {
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            return electrumBackendService.getTransactionMerkle(txHash, height);
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use blockchain.transaction.get_merkle");
+    }
+
+    @JsonRpcMethod("blockchain.transaction.id_from_pos")
+    public Object getTransactionIdFromPos(@JsonRpcParam("height") int height, @JsonRpcParam("tx_pos") int txPos, @JsonRpcParam("merkle") @JsonRpcOptional Boolean merkle) {
+        checkVersionNegotiated();
+        if(electrumBackendService != null) {
+            return electrumBackendService.getTransactionIdFromPos(height, txPos, merkle);
+        }
+
+        throw new UnsupportedOperationException("Configure backendElectrumServer to use blockchain.transaction.id_from_pos");
+    }
+
     @JsonRpcMethod("blockchain.transaction.broadcast")
     public String broadcastTransaction(@JsonRpcParam("raw_tx") String rawTx) throws BitcoindIOException, BroadcastFailedException {
+        checkVersionNegotiated();
+        if(bitcoindClient == null) {
+            throw new UnsupportedOperationException("Configure coreServer to use blockchain.transaction.broadcast");
+        }
+
         try {
             return bitcoindClient.getBitcoindService().sendRawTransaction(rawTx, 0d);
         } catch(JsonRpcException e) {
@@ -197,40 +478,160 @@ public class ElectrumServerService {
         }
     }
 
+    @JsonRpcMethod("blockchain.transaction.broadcast_package")
+    public Object broadcastTransactionPackage(@JsonRpcParam("raw_txs") String[] rawTxes, @JsonRpcParam("verbose") @JsonRpcOptional Boolean verbose) throws BitcoindIOException, BroadcastFailedException {
+        checkVersionNegotiated();
+        if(bitcoindClient == null) {
+            throw new UnsupportedOperationException("Configure coreServer to use blockchain.transaction.broadcast_package");
+        }
+
+        try {
+            if(verbose == null || !verbose) {
+                PackageResult result = bitcoindClient.getBitcoindService().submitPackage(rawTxes, null, null);
+                return PackageResultSummary.fromPackageResult(result);
+            } else {
+                return bitcoindClient.getBitcoindService().submitPackage(rawTxes, null, null);
+            }
+        } catch(JsonRpcException e) {
+            throw new BroadcastFailedException(e.getErrorMessage());
+        } catch(IllegalStateException e) {
+            throw new BitcoindIOException(e);
+        }
+    }
+
     @JsonRpcMethod("blockchain.silentpayments.subscribe")
-    public String subscribeSilentPayments(@JsonRpcParam("scan_private_key") String scanPrivateKey, @JsonRpcParam("spend_public_key") String spendPublicKey, @JsonRpcParam("start") @JsonRpcOptional Long start) {
-        SilentPaymentScanAddress silentPaymentScanAddress = getSilentPaymentScanAddress(scanPrivateKey, spendPublicKey);
-        requestHandler.subscribeSilentPaymentsAddress(silentPaymentScanAddress);
+    public SilentPaymentsSubscription subscribeSilentPayments(@JsonRpcParam("scan_private_key") String scanPrivateKey, @JsonRpcParam("spend_public_key") String spendPublicKey, @JsonRpcParam("start") @JsonRpcOptional Object start, @JsonRpcParam("labels") @JsonRpcOptional Integer[] labels) throws InvalidParamsException {
+        checkVersionNegotiated();
+        SilentPaymentScanAddress silentPaymentScanAddress = parseScanAddress(scanPrivateKey, spendPublicKey);
+        Set<Integer> labelSet = parseLabels(labels);
 
-        int startHeight = getStartHeight(start);
-        indexQuerier.startHistoryScan(silentPaymentScanAddress, startHeight, null, new WeakReference<>(requestHandler));
+        int maxSubscriptions = Config.get().getScan().getMaxSubscriptions();
+        if(!requestHandler.isSilentPaymentsAddressSubscribed(silentPaymentScanAddress.toString()) && requestHandler.getSilentPaymentsSubscriptionCount() >= maxSubscriptions) {
+            throw new InvalidParamsException("subscription limit reached (" + maxSubscriptions + ") for this connection");
+        }
 
-        return silentPaymentScanAddress.getAddress();
+        int[] heightRange = getHeightRange(start);
+        int requestedStart = heightRange[0];
+        Integer endHeight = heightRange.length > 1 ? heightRange[1] : null;
+
+        int effectiveStart = requestedStart;
+        SilentPaymentAddressSubscription existing = requestHandler.getSilentPaymentsAddressSubscription(silentPaymentScanAddress.toString());
+        if(existing != null && existing.getStartHeight() < requestedStart) {
+            effectiveStart = existing.getStartHeight();
+        }
+
+        int startHeight = effectiveStart;
+        requestHandler.subscribeSilentPaymentsAddress(silentPaymentScanAddress, labelSet, startHeight);
+
+        requestHandler.runAfterResponse(() -> {
+            SilentPaymentAddressSubscription subscription = requestHandler.getSilentPaymentsAddressSubscription(silentPaymentScanAddress.toString());
+            if(subscription == null) {
+                return;
+            }
+            subscription.setActive(true);
+            indexQuerier.startHistoryScan(silentPaymentScanAddress, startHeight, endHeight, subscription, new WeakReference<>(requestHandler), true);
+        });
+
+        return new SilentPaymentsSubscription(silentPaymentScanAddress.getAddress(), labelSet.toArray(new Integer[0]), startHeight);
     }
 
     @JsonRpcMethod("blockchain.silentpayments.unsubscribe")
-    public String unsubscribeSilentPayments(@JsonRpcParam("scan_private_key") String scanPrivateKey, @JsonRpcParam("spend_public_key") String spendPublicKey) {
-        SilentPaymentScanAddress silentPaymentScanAddress = getSilentPaymentScanAddress(scanPrivateKey, spendPublicKey);
+    public String unsubscribeSilentPayments(@JsonRpcParam("scan_private_key") String scanPrivateKey, @JsonRpcParam("spend_public_key") String spendPublicKey) throws InvalidParamsException {
+        checkVersionNegotiated();
+        SilentPaymentScanAddress silentPaymentScanAddress = parseScanAddress(scanPrivateKey, spendPublicKey);
         requestHandler.unsubscribeSilentPaymentsAddress(silentPaymentScanAddress);
 
         return silentPaymentScanAddress.getAddress();
     }
 
-    private static SilentPaymentScanAddress getSilentPaymentScanAddress(String scanPrivateKey, String spendPublicKey) {
-        ECKey scanKey = ECKey.fromPrivate(Utils.hexToBytes(scanPrivateKey));
-        ECKey spendKey = ECKey.fromPublicOnly(Utils.hexToBytes(spendPublicKey));
-        return SilentPaymentScanAddress.from(scanKey, spendKey);
+    private static SilentPaymentScanAddress parseScanAddress(String scanPrivateKey, String spendPublicKey) throws InvalidParamsException {
+        byte[] scanBytes;
+        byte[] spendBytes;
+        try {
+            scanBytes = Utils.hexToBytes(scanPrivateKey);
+            spendBytes = Utils.hexToBytes(spendPublicKey);
+        } catch(IllegalArgumentException e) {
+            throw new InvalidParamsException("scan_private_key or spend_public_key is not valid hex", e);
+        }
+        if(scanBytes.length != 32) {
+            throw new InvalidParamsException("scan_private_key must be 32 bytes");
+        }
+        if(spendBytes.length != 33) {
+            throw new InvalidParamsException("spend_public_key must be 33 bytes (compressed)");
+        }
+        try {
+            ECKey scanKey = ECKey.fromPrivate(scanBytes);
+            ECKey spendKey = ECKey.fromPublicOnly(spendBytes);
+            return SilentPaymentScanAddress.from(scanKey, spendKey);
+        } catch(IllegalArgumentException e) {
+            throw new InvalidParamsException("invalid scan/spend key: " + e.getMessage(), e);
+        }
     }
 
-    private int getStartHeight(Long start) {
-        int startHeight = 0;
-        if(start != null) {
-            if(start > Transaction.MAX_BLOCK_LOCKTIME) {
-                startHeight = bitcoindClient.findBlockByTimestamp(start);
-            } else if(start > 0) {
-                startHeight = start.intValue();
+    private int[] getHeightRange(Object start) throws InvalidParamsException {
+        if(start == null) {
+            return new int[] { 0 };
+        }
+
+        if(start instanceof String s) {
+            if(!s.contains("-")) {
+                throw new InvalidParamsException("start string must be of the form 'FROM-TO'");
+            }
+            String[] parts = s.split("-", 2);
+            int from;
+            int to;
+            try {
+                from = Integer.parseInt(parts[0]);
+                to = Integer.parseInt(parts[1]);
+            } catch(NumberFormatException e) {
+                throw new InvalidParamsException("start range must contain integer block heights", e);
+            }
+            if(from < 0 || to < from) {
+                throw new InvalidParamsException("start range must satisfy 0 <= from <= to");
+            }
+            int tip = bitcoindClient != null && bitcoindClient.getTip() != null ? bitcoindClient.getTip().height() : Integer.MAX_VALUE;
+            if(to > tip) {
+                throw new InvalidParamsException("start range 'to' (" + to + ") exceeds tip (" + tip + ")");
+            }
+            return new int[] { from, to };
+        }
+
+        if(start instanceof Number n) {
+            long startLong = n.longValue();
+            if(startLong < 0) {
+                throw new InvalidParamsException("start must be non-negative");
+            }
+            if(startLong > Transaction.MAX_BLOCK_LOCKTIME) {
+                if(bitcoindClient == null) {
+                    throw new InvalidParamsException("timestamp start requires coreServer to be configured");
+                }
+                return new int[] { bitcoindClient.findBlockByTimestamp(startLong) };
+            }
+            return new int[] { (int)startLong };
+        }
+
+        throw new InvalidParamsException("start must be an integer or 'FROM-TO' string");
+    }
+
+    private Set<Integer> parseLabels(Integer[] labels) throws InvalidParamsException {
+        int maxLabels = Config.get().getScan().getMaxLabels();
+        if(labels != null && labels.length > maxLabels) {
+            throw new InvalidParamsException("labels array exceeds " + maxLabels + " entries");
+        }
+
+        SortedSet<Integer> labelSet = new TreeSet<>();
+        labelSet.add(0);
+        if(labels != null) {
+            for(Integer label : labels) {
+                if(label == null) {
+                    continue;
+                }
+                if(label <= 0) {
+                    throw new InvalidParamsException("label must satisfy 0 < label < 2^31, got " + label);
+                }
+                labelSet.add(label);
             }
         }
-        return startHeight;
+        return Collections.unmodifiableSortedSet(labelSet);
     }
 }

src/main/java/com/sparrowwallet/frigate/electrum/ElectrumTransport.java

@@ -0,0 +1,326 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.github.arteam.simplejsonrpc.client.Transport;
+import com.github.arteam.simplejsonrpc.server.JsonRpcServer;
+import com.google.common.net.HostAndPort;
+import com.sparrowwallet.frigate.io.Protocol;
+import com.sparrowwallet.frigate.io.SslUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.SocketFactory;
+import javax.net.ssl.SSLSocketFactory;
+import java.io.*;
+import java.net.*;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ElectrumTransport implements Transport, Closeable {
+    private static final Logger log = LoggerFactory.getLogger(ElectrumTransport.class);
+
+    private final HostAndPort electrumServer;
+    private final Protocol protocol;
+    private Socket socket;
+    private String response;
+
+    private boolean firstRead = true;
+
+    private final CountDownLatch readReadySignal = new CountDownLatch(1);
+
+    private final ReentrantLock readLock = new ReentrantLock();
+    private final Condition readingCondition = readLock.newCondition();
+
+    private final ReentrantLock clientRequestLock = new ReentrantLock();
+    private volatile boolean running = false;
+    private volatile boolean reading = true;
+    private volatile boolean closed = false;
+    private Exception lastException;
+
+    private static final Pattern ID_PATTERN = Pattern.compile("\"id\"\\s*:\\s*(\\d+)");
+    private static final JsonFactory JSON_FACTORY = new JsonFactory();
+
+    private final JsonRpcServer jsonRpcServer = new JsonRpcServer();
+    private final Object subscriptionService;
+
+    private PrintWriter out;
+    private BufferedReader in;
+
+    public ElectrumTransport(HostAndPort electrumServer, Protocol protocol, Object subscriptionService) {
+        this.electrumServer = electrumServer;
+        this.protocol = protocol;
+        this.subscriptionService = subscriptionService;
+    }
+
+    public void connect() {
+        try {
+            String host = electrumServer.getHost();
+            int port = electrumServer.hasPort() ? electrumServer.getPort() : protocol.getDefaultPort();
+
+            SocketFactory socketFactory;
+            if(protocol == Protocol.SSL) {
+                SSLSocketFactory sslSocketFactory = SslUtil.getTrustAllSocketFactory();
+                if(sslSocketFactory == null) {
+                    log.error("Could not create SSL socket factory for Electrum server: " + host);
+                    return;
+                }
+                socketFactory = sslSocketFactory;
+            } else {
+                socketFactory = SocketFactory.getDefault();
+            }
+
+            this.socket = socketFactory.createSocket();
+            this.socket.connect(new InetSocketAddress(host, port));
+            this.socket.setSoTimeout(30000); // 30 second timeout for reads
+            this.out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8)));
+            this.in = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8));
+            this.running = true;
+        } catch(UnknownHostException e) {
+            log.error("Unknown host: " + electrumServer.getHost());
+        } catch(IOException e) {
+            log.error("Error connecting to Electrum server: " + electrumServer.getHost());
+        }
+    }
+
+    @Override
+    public String pass(String request) throws IOException {
+        Set<String> sentIdSet = extractIdSet(request);
+        clientRequestLock.lock();
+        try {
+            writeRequest(request);
+
+            String recv;
+            Set<String> recvIdSet;
+            do {
+                recv = readResponse();
+                recvIdSet = extractIdSet(recv);
+                if(!sentIdSet.equals(recvIdSet)) {
+                    log.info("Discarding stale response with ids " + recvIdSet + " (expected " + sentIdSet + ")");
+                }
+            } while(!sentIdSet.equals(recvIdSet));
+
+            return recv;
+        } finally {
+            clientRequestLock.unlock();
+        }
+    }
+
+    protected void writeRequest(String request) throws IOException {
+        log.debug("> " + request);
+
+        if(out == null) {
+            throw new IllegalStateException("Socket connection has not been established.");
+        }
+
+        out.println(request);
+        out.flush();
+    }
+
+    private String readResponse() throws IOException {
+        if(firstRead) {
+            try {
+                //Ensure read thread has started
+                if(!readReadySignal.await(2, TimeUnit.SECONDS)) {
+                    throw new IOException("Read thread did not start");
+                }
+            } catch(InterruptedException e) {
+                throw new IOException("Read ready await interrupted");
+            }
+        }
+
+        readLock.lock();
+        try {
+            if(firstRead) {
+                readingCondition.signal();
+                firstRead = false;
+            }
+
+            while(reading && running) {
+                try {
+                    readingCondition.await();
+                } catch(InterruptedException e) {
+                    //Restore interrupt status and break
+                    Thread.currentThread().interrupt();
+                    break;
+                }
+            }
+
+            if(lastException != null) {
+                throw new IOException("Error reading response: " + lastException.getMessage(), lastException);
+            }
+
+            if(!running) {
+                throw new IOException("Transport closed");
+            }
+
+            reading = true;
+
+            readingCondition.signal();
+            return response;
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    public void readInputLoop() throws Exception {
+        //Wait for first RPC request before starting to read. The lock must be acquired before
+        //signaling readiness so readResponse() blocks until we reach the atomic await/unlock.
+        readLock.lock();
+        try {
+            readReadySignal.countDown();
+            if(running) {
+                readingCondition.await();
+            }
+        } catch(InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return;
+        } finally {
+            readLock.unlock();
+        }
+
+        while(running) {
+            try {
+                String received = readInputStream(in);
+                if(isNotification(received)) {
+                    jsonRpcServer.handle(received, subscriptionService);
+                } else {
+                    deliverResponse(received);
+                }
+            } catch(InterruptedException e) {
+                //Restore interrupt status and continue
+                Thread.currentThread().interrupt();
+            } catch(Exception e) {
+                if(!closed) {
+                    log.trace("Connection error while reading", e);
+                }
+                if(running) {
+                    signalException(e);
+                    //Allow this thread to terminate as we will need to reconnect with a new transport anyway
+                    running = false;
+                }
+            }
+        }
+    }
+
+    private void deliverResponse(String received) throws InterruptedException {
+        readLock.lock();
+        try {
+            response = received;
+            reading = false;
+            readingCondition.signal();
+            while(!reading && running) {
+                readingCondition.await();
+            }
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    private void signalException(Exception e) {
+        readLock.lock();
+        try {
+            lastException = e;
+            reading = false;
+            readingCondition.signal();
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    protected String readInputStream(BufferedReader in) throws IOException {
+        String response = readLine(in);
+
+        if(response == null) {
+            throw new IOException("Could not connect to server " + electrumServer);
+        }
+
+        log.debug("< " + response);
+
+        return response;
+    }
+
+    private String readLine(BufferedReader in) throws IOException {
+        while(!socket.isClosed()) {
+            try {
+                return in.readLine();
+            } catch(SocketTimeoutException e) {
+                //ignore and continue
+            }
+        }
+
+        return null;
+    }
+
+    private static boolean isNotification(String json) {
+        try(JsonParser parser = JSON_FACTORY.createParser(json)) {
+            if(parser.nextToken() != JsonToken.START_OBJECT) {
+                return false;
+            }
+            while(parser.nextToken() == JsonToken.FIELD_NAME) {
+                String field = parser.currentName();
+                JsonToken value = parser.nextToken();
+                if("method".equals(field)) {
+                    return value == JsonToken.VALUE_STRING;
+                }
+                parser.skipChildren();
+            }
+            return false;
+        } catch(Exception e) {
+            log.warn("Could not parse JSON-RPC message from backend: " + e.getMessage());
+            return false;
+        }
+    }
+
+    public Exception getLastException() {
+        return lastException;
+    }
+
+    @Override
+    public void close() throws IOException {
+        running = false;
+        closed = true;
+
+        readLock.lock();
+        try {
+            readingCondition.signalAll();
+        } finally {
+            readLock.unlock();
+        }
+
+        if(out != null) {
+            out.close();
+        }
+        if(in != null) {
+            in.close();
+        }
+        if(socket != null) {
+            socket.close();
+        }
+    }
+
+    public boolean isClosed() {
+        return closed;
+    }
+
+    private static Set<String> extractIdSet(String json) {
+        if(json == null || json.isEmpty()) {
+            return Collections.emptySet();
+        }
+        Matcher m = ID_PATTERN.matcher(json);
+        Set<String> ids = new LinkedHashSet<>();
+        while(m.find()) {
+            ids.add(m.group(1));
+        }
+        return ids;
+    }
+}

src/main/java/com/sparrowwallet/frigate/electrum/InvalidParamsException.java

@@ -0,0 +1,14 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcError;
+
+@JsonRpcError(code = -32602, message = "Invalid params")
+public class InvalidParamsException extends Exception {
+    public InvalidParamsException(String message) {
+        super(message);
+    }
+
+    public InvalidParamsException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

src/main/java/com/sparrowwallet/frigate/electrum/PackageResultSummary.java

@@ -0,0 +1,15 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.sparrowwallet.frigate.bitcoind.PackageResult;
+
+public record PackageResultSummary(boolean success, PackageResultError[] errors) {
+    public record PackageResultError(String txid, String error) {}
+
+    public static PackageResultSummary fromPackageResult(PackageResult result) {
+        return new PackageResultSummary("success".equals(result.package_msg()),
+                result.txResults().values().stream()
+                    .filter(r -> r.error() != null)
+                    .map(r -> new PackageResultError(r.txid(), r.error()))
+                    .toArray(PackageResultError[]::new));
+    }
+}

src/main/java/com/sparrowwallet/frigate/electrum/RequestHandler.java

@@ -8,8 +8,11 @@ import com.sparrowwallet.drongo.silentpayments.SilentPaymentScanAddress;
 import com.sparrowwallet.frigate.Frigate;
 import com.sparrowwallet.frigate.SubscriptionStatus;
 import com.sparrowwallet.frigate.bitcoind.BitcoindClient;
-import com.sparrowwallet.frigate.bitcoind.BlockReorgEvent;
+import com.sparrowwallet.frigate.bitcoind.BlockReorgSyncStart;
+import com.sparrowwallet.frigate.bitcoind.BlockReorgSyncComplete;
 import com.sparrowwallet.frigate.index.*;
+import com.sparrowwallet.frigate.io.Config;
+import com.sparrowwallet.frigate.io.Server;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -17,28 +20,49 @@ import java.io.*;
 import java.lang.ref.WeakReference;
 import java.net.Socket;
 import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
+import java.util.ArrayDeque;
+import java.util.Deque;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
-public class RequestHandler implements Runnable, SubscriptionStatus {
+public class RequestHandler implements Runnable, SubscriptionStatus, Thread.UncaughtExceptionHandler {
     private static final Logger log = LoggerFactory.getLogger(RequestHandler.class);
     private final Socket clientSocket;
     private final ElectrumServerService electrumServerService;
     private final JsonRpcServer rpcServer = new JsonRpcServer();
     private final AtomicBoolean disconnected = new AtomicBoolean(false);
+    private final ElectrumTransport backendTransport;
+    private final Thread reader;
 
     private boolean connected;
-    private boolean headersSubscribed;
-    private final Set<String> scriptHashesSubscribed = new HashSet<>();
-    private final Map<String, SilentPaymentAddressSubscription> silentPaymentsAddressesSubscribed = new HashMap<>();
+    private volatile boolean headersSubscribed;
+    private final Set<String> scriptHashesSubscribed = ConcurrentHashMap.newKeySet();
+    private final Map<String, SilentPaymentAddressSubscription> silentPaymentsAddressesSubscribed = new ConcurrentHashMap<>();
+    private final Deque<Runnable> postResponseTasks = new ArrayDeque<>();
+
+    private final ReentrantLock writeLock = new ReentrantLock();
+    private volatile PrintWriter out;
+    private final ElectrumNotificationService notificationService;
 
     public RequestHandler(Socket clientSocket, BitcoindClient bitcoindClient, IndexQuerier indexQuerier) {
         this.clientSocket = clientSocket;
-        this.electrumServerService = new ElectrumServerService(bitcoindClient, this, indexQuerier);
+        Server backendServer = Config.get().getServer().getBackendElectrumServerObj();
+        if(backendServer != null) {
+            this.backendTransport = new ElectrumTransport(backendServer.getHostAndPort(), backendServer.getProtocol(), new BackendSubscriptionService());
+            this.reader = Thread.ofVirtual().name("BackendServerReadThread-" + System.identityHashCode(this)).unstarted(new ReadRunnable(backendTransport));
+            reader.setUncaughtExceptionHandler(this);
+        } else {
+            this.backendTransport = null;
+            this.reader = null;
+        }
+        this.electrumServerService = new ElectrumServerService(bitcoindClient, this, indexQuerier, backendTransport);
+        this.notificationService = new JsonRpcClient(new ElectrumNotificationTransport(this)).onDemand(ElectrumNotificationService.class);
     }
 
     public void run() {
@@ -46,28 +70,84 @@ public class RequestHandler implements Runnable, SubscriptionStatus {
         this.connected = true;
 
         try {
-            InputStream input  = clientSocket.getInputStream();
+            InputStream input = clientSocket.getInputStream();
             BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8));
 
             OutputStream output = clientSocket.getOutputStream();
-            PrintWriter out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8)));
+            this.out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8)));
+
+            connectBackendTransport();
 
             while(true) {
+                postResponseTasks.clear();
+
                 String request = reader.readLine();
                 if(request == null) {
                     break;
                 }
 
+                // Skip requests with null bytes or other control characters
+                if(request.indexOf(0) >= 0 || request.chars().anyMatch(c -> c < 32 && c != '\t' && c != '\r' && c != '\n')) {
+                    log.warn("Skipping malformed request with control characters");
+                    continue;
+                }
+
                 String response = rpcServer.handle(request, electrumServerService);
-                out.println(response);
-                out.flush();
+                writeLine(response);
+
+                runPostResponseTasks();
             }
         } catch(IOException e) {
-            log.error("Could not communicate with client socket", e);
+            log.debug("Could not communicate with client socket: {}", e.getMessage());
         } finally {
+            closeBackendTransport();
             this.connected = false;
             this.disconnected.set(true);
             Frigate.getEventBus().unregister(this);
+
+            try {
+                clientSocket.close();
+            } catch(IOException e) {
+                log.error("Error closing client socket", e);
+            }
+        }
+    }
+
+    public void writeLine(String line) {
+        writeLock.lock();
+        try {
+            PrintWriter writer = out;
+            if(writer == null) {
+                return;
+            }
+            writer.println(line);
+            writer.flush();
+        } finally {
+            writeLock.unlock();
+        }
+    }
+
+    private void connectBackendTransport() {
+        if(backendTransport != null) {
+            backendTransport.connect();
+        }
+
+        if(reader != null && !reader.isAlive()) {
+            reader.start();
+        }
+    }
+
+    private void closeBackendTransport() {
+        if(backendTransport != null) {
+            try {
+                backendTransport.close();
+            } catch(IOException e) {
+                log.error("Error closing transport", e);
+            }
+        }
+
+        if(reader != null && reader.isAlive()) {
+            reader.interrupt();
         }
     }
 
@@ -89,17 +169,51 @@ public class RequestHandler implements Runnable, SubscriptionStatus {
         scriptHashesSubscribed.add(scriptHash);
     }
 
+    public void unsubscribeScriptHash(String scriptHash) {
+        scriptHashesSubscribed.remove(scriptHash);
+    }
+
     @Override
     public boolean isScriptHashSubscribed(String scriptHash) {
         return scriptHashesSubscribed.contains(scriptHash);
     }
 
-    public void subscribeSilentPaymentsAddress(SilentPaymentScanAddress silentPaymentsScanAddress) {
-        silentPaymentsAddressesSubscribed.put(silentPaymentsScanAddress.toString(), new SilentPaymentAddressSubscription(silentPaymentsScanAddress));
+    public void subscribeSilentPaymentsAddress(SilentPaymentScanAddress silentPaymentsScanAddress, Set<Integer> labelSet, int startHeight) {
+        SilentPaymentAddressSubscription previous = silentPaymentsAddressesSubscribed.get(silentPaymentsScanAddress.toString());
+        if(previous != null) {
+            previous.invalidateInFlightScans();
+        }
+        silentPaymentsAddressesSubscribed.put(silentPaymentsScanAddress.toString(), new SilentPaymentAddressSubscription(silentPaymentsScanAddress, labelSet, startHeight));
     }
 
     public void unsubscribeSilentPaymentsAddress(SilentPaymentScanAddress silentPaymentsScanAddress) {
-        silentPaymentsAddressesSubscribed.remove(silentPaymentsScanAddress.toString());
+        SilentPaymentAddressSubscription previous = silentPaymentsAddressesSubscribed.remove(silentPaymentsScanAddress.toString());
+        if(previous != null) {
+            previous.invalidateInFlightScans();
+        }
+    }
+
+    public SilentPaymentAddressSubscription getSilentPaymentsAddressSubscription(String silentPaymentsAddress) {
+        return silentPaymentsAddressesSubscribed.get(silentPaymentsAddress);
+    }
+
+    public void runAfterResponse(Runnable task) {
+        postResponseTasks.add(task);
+    }
+
+    private void runPostResponseTasks() {
+        while(!postResponseTasks.isEmpty()) {
+            Runnable task = postResponseTasks.poll();
+            try {
+                task.run();
+            } catch(Exception e) {
+                log.error("Error running post-response task", e);
+            }
+        }
+    }
+
+    public int getSilentPaymentsSubscriptionCount() {
+        return silentPaymentsAddressesSubscribed.size();
     }
 
     @Override
@@ -116,18 +230,18 @@ public class RequestHandler implements Runnable, SubscriptionStatus {
     @Subscribe
     public void newBlock(ElectrumBlockHeader electrumBlockHeader) {
         if(isHeadersSubscribed()) {
-            ElectrumNotificationTransport electrumNotificationTransport = new ElectrumNotificationTransport(clientSocket);
-            JsonRpcClient jsonRpcClient = new JsonRpcClient(electrumNotificationTransport);
-            jsonRpcClient.onDemand(ElectrumNotificationService.class).notifyHeaders(electrumBlockHeader);
+            notifyHeaders(electrumBlockHeader);
         }
     }
 
+    void notifyHeaders(ElectrumBlockHeader electrumBlockHeader) {
+        notificationService.notifyHeaders(electrumBlockHeader);
+    }
+
     @Subscribe
     public void scriptHashStatus(ScriptHashStatus scriptHashStatus) {
         if(isScriptHashSubscribed(scriptHashStatus.scriptHash())) {
-            ElectrumNotificationTransport electrumNotificationTransport = new ElectrumNotificationTransport(clientSocket);
-            JsonRpcClient jsonRpcClient = new JsonRpcClient(electrumNotificationTransport);
-            jsonRpcClient.onDemand(ElectrumNotificationService.class).notifyScriptHash(scriptHashStatus.scriptHash(), scriptHashStatus.status());
+            notificationService.notifyScriptHash(scriptHashStatus.scriptHash(), scriptHashStatus.status());
         }
     }
 
@@ -135,20 +249,24 @@ public class RequestHandler implements Runnable, SubscriptionStatus {
     public void silentPaymentsNotification(SilentPaymentsNotification notification) {
         if(isSilentPaymentsAddressSubscribed(notification.subscription().address()) && notification.status() == this) {
             SilentPaymentAddressSubscription subscription = silentPaymentsAddressesSubscribed.get(notification.subscription().address());
-            subscription.setHighestBlockHeight(notification.history().stream().mapToInt(TxEntry::getHeight).max().orElse(subscription.getHighestBlockHeight()));
+            if(!subscription.isActive()) {
+                return;
+            }
+            notification.history().stream().mapToInt(SilentPaymentsTxEntry::getHeight).filter(h -> h > 0).max().ifPresent(subscription::accumulateMaxBlockHeight);
             subscription.getMempoolTxids().addAll(notification.history().stream().filter(txEntry -> txEntry.height <= 0).map(txEntry -> Sha256Hash.wrap(txEntry.tx_hash)).collect(Collectors.toSet()));
 
-            ElectrumNotificationTransport electrumNotificationTransport = new ElectrumNotificationTransport(clientSocket);
-            JsonRpcClient jsonRpcClient = new JsonRpcClient(electrumNotificationTransport);
-            jsonRpcClient.onDemand(ElectrumNotificationService.class).notifySilentPayments(notification.subscription(), notification.progress(), notification.history());
+            List<SilentPaymentsTxEntry> deliverable = notification.history().stream()
+                    .filter(txEntry -> txEntry.height <= 0 || !subscription.getMempoolTxids().contains(Sha256Hash.wrap(txEntry.tx_hash))).toList();
+
+            notificationService.notifySilentPayments(notification.subscription(), notification.progress(), deliverable);
         }
     }
 
     @Subscribe
     public void silentPaymentsBlocksIndexUpdate(SilentPaymentsBlocksIndexUpdate update) {
         for(SilentPaymentAddressSubscription subscription : silentPaymentsAddressesSubscribed.values()) {
-            if(update.fromBlockHeight() > subscription.getHighestBlockHeight()) {
-                electrumServerService.getIndexQuerier().startHistoryScan(subscription.getAddress(), update.fromBlockHeight(), null, new WeakReference<>(this), false);
+            if(subscription.isActive() && !subscription.isPendingHistoricalRescan() && update.fromBlockHeight() > subscription.getHighestBlockHeight()) {
+                electrumServerService.getIndexQuerier().startHistoryScan(subscription.getAddress(), update.fromBlockHeight(), null, subscription, new WeakReference<>(this), false);
             }
         }
     }
@@ -156,21 +274,57 @@ public class RequestHandler implements Runnable, SubscriptionStatus {
     @Subscribe
     public void silentPaymentsMempoolIndexAdded(SilentPaymentsMempoolIndexAdded added) {
         for(SilentPaymentAddressSubscription subscription : silentPaymentsAddressesSubscribed.values()) {
-            electrumServerService.getIndexQuerier().startMempoolScan(subscription.getAddress(), null, null, new WeakReference<>(this));
+            if(subscription.isActive()) {
+                electrumServerService.getIndexQuerier().startMempoolScan(subscription.getAddress(), null, null, added.getTxids(), subscription, new WeakReference<>(this));
+            }
         }
     }
 
     @Subscribe
-    public void silentPaymentsMempoolIndexRemoved(SilentPaymentsMempoolIndexRemoved removed) {
+    public void blockReorgSyncStart(BlockReorgSyncStart event) {
+        int reorgPoint = event.reorgStartHeight() - 1;
         for(SilentPaymentAddressSubscription subscription : silentPaymentsAddressesSubscribed.values()) {
-            subscription.getMempoolTxids().removeAll(removed.getTxids());
+            subscription.invalidateInFlightScans();
+            subscription.accumulateMinBlockHeight(reorgPoint);
+            if(subscription.isActive() && !subscription.isHistoricalComplete()) {
+                subscription.markPendingHistoricalRescan();
+            }
         }
     }
 
     @Subscribe
-    public void blockReorgEvent(BlockReorgEvent event) {
+    public void blockReorgSyncComplete(BlockReorgSyncComplete event) {
         for(SilentPaymentAddressSubscription subscription : silentPaymentsAddressesSubscribed.values()) {
-            subscription.setHighestBlockHeight(event.startHeight() - 1);
+            if(subscription.isActive() && subscription.consumePendingHistoricalRescan()) {
+                int scanFrom = subscription.getHighestBlockHeight() + 1;
+                electrumServerService.getIndexQuerier().startHistoryScan(subscription.getAddress(), scanFrom, null, subscription, new WeakReference<>(this), true);
+            }
+        }
+    }
+
+    @Override
+    public void uncaughtException(Thread t, Throwable e) {
+        log.error("Uncaught exception in thread " + t.getName(), e);
+    }
+
+    public static class ReadRunnable implements Runnable {
+        private final ElectrumTransport electrumTransport;
+
+        public ReadRunnable(ElectrumTransport electrumTransport) {
+            this.electrumTransport = electrumTransport;
+        }
+
+        @Override
+        public void run() {
+            try {
+                electrumTransport.readInputLoop();
+
+                if(electrumTransport.getLastException() != null && !electrumTransport.isClosed()) {
+                    log.error("Connection to Electrum server lost", electrumTransport.getLastException());
+                }
+            } catch(Exception e) {
+                log.debug("Read thread terminated", e);
+            }
         }
     }
 }

src/main/java/com/sparrowwallet/frigate/electrum/ScriptHashBalance.java

@@ -0,0 +1,3 @@
+package com.sparrowwallet.frigate.electrum;
+
+public record ScriptHashBalance(long confirmed, long unconfirmed) {}

src/main/java/com/sparrowwallet/frigate/electrum/ServerFeatures.java

@@ -0,0 +1,31 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import java.util.List;
+import java.util.Map;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public record ServerFeatures(Map<String, HostInfo> hosts, String genesis_hash, String hash_function, String server_version, String protocol_max, String protocol_min, Integer pruning, List<Integer> silent_payments) {
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public record HostInfo(Integer tcp_port, Integer ssl_port) {
+        //Some non-conformant Electrum servers return hosts as {"host": <port>} instead of
+        //{"host": {"tcp_port": <port>, "ssl_port": <port>}}. Accept that shape and treat the
+        //bare number as the tcp_port.
+        @JsonCreator(mode = JsonCreator.Mode.DELEGATING)
+        public static HostInfo fromTcpPort(int tcpPort) {
+            return new HostInfo(tcpPort, null);
+        }
+    }
+
+    public ServerFeatures withSilentPayments(List<Integer> versions) {
+        return new ServerFeatures(hosts, genesis_hash, hash_function, server_version, protocol_max, protocol_min, pruning, versions);
+    }
+
+    public ServerFeatures withHosts(Map<String, HostInfo> hosts) {
+        return new ServerFeatures(hosts, genesis_hash, hash_function, server_version, protocol_max, protocol_min, pruning, silent_payments);
+    }
+}

src/main/java/com/sparrowwallet/frigate/electrum/ServerPeer.java

@@ -0,0 +1,8 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+
+import java.util.List;
+
+@JsonFormat(shape = JsonFormat.Shape.ARRAY)
+public record ServerPeer(String ip, String hostname, List<String> features) {}

src/main/java/com/sparrowwallet/frigate/electrum/SilentPaymentAddressSubscription.java

@@ -3,28 +3,89 @@ package com.sparrowwallet.frigate.electrum;
 import com.sparrowwallet.drongo.protocol.Sha256Hash;
 import com.sparrowwallet.drongo.silentpayments.SilentPaymentScanAddress;
 
-import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BooleanSupplier;
 
 public class SilentPaymentAddressSubscription {
     private final SilentPaymentScanAddress address;
-    private int highestBlockHeight;
-    private final Set<Sha256Hash> mempoolTxids = new HashSet<>();
+    private final Set<Integer> labels;
+    private final int startHeight;
+    private final AtomicInteger highestBlockHeight = new AtomicInteger();
+    private final Set<Sha256Hash> mempoolTxids = ConcurrentHashMap.newKeySet();
+    private volatile boolean active;
+    private volatile boolean historicalComplete;
+    private final AtomicBoolean pendingHistoricalRescan = new AtomicBoolean(false);
+    private final AtomicLong scanEpoch = new AtomicLong();
 
-    public SilentPaymentAddressSubscription(SilentPaymentScanAddress address) {
+    public SilentPaymentAddressSubscription(SilentPaymentScanAddress address, Set<Integer> labels, int startHeight) {
         this.address = address;
+        this.labels = labels;
+        this.startHeight = startHeight;
+    }
+
+    public int getStartHeight() {
+        return startHeight;
+    }
+
+    public boolean isActive() {
+        return active;
+    }
+
+    public void setActive(boolean active) {
+        this.active = active;
+    }
+
+    public BooleanSupplier captureScanCancellation() {
+        long captured = scanEpoch.get();
+        return () -> scanEpoch.get() != captured;
+    }
+
+    public void invalidateInFlightScans() {
+        scanEpoch.incrementAndGet();
+    }
+
+    public boolean isHistoricalComplete() {
+        return historicalComplete;
+    }
+
+    public void markHistoricalComplete() {
+        historicalComplete = true;
+    }
+
+    public void markPendingHistoricalRescan() {
+        pendingHistoricalRescan.set(true);
+    }
+
+    public boolean isPendingHistoricalRescan() {
+        return pendingHistoricalRescan.get();
+    }
+
+    public boolean consumePendingHistoricalRescan() {
+        return pendingHistoricalRescan.getAndSet(false);
     }
 
     public SilentPaymentScanAddress getAddress() {
         return address;
     }
 
-    public int getHighestBlockHeight() {
-        return highestBlockHeight;
+    public Set<Integer> getLabels() {
+        return labels;
     }
 
-    public void setHighestBlockHeight(int highestBlockHeight) {
-        this.highestBlockHeight = highestBlockHeight;
+    public int getHighestBlockHeight() {
+        return highestBlockHeight.get();
+    }
+
+    public void accumulateMaxBlockHeight(int candidate) {
+        highestBlockHeight.accumulateAndGet(candidate, Math::max);
+    }
+
+    public void accumulateMinBlockHeight(int candidate) {
+        highestBlockHeight.accumulateAndGet(candidate, Math::min);
     }
 
     public Set<Sha256Hash> getMempoolTxids() {

src/main/java/com/sparrowwallet/frigate/electrum/SilentPaymentsNotification.java

@@ -1,10 +1,9 @@
 package com.sparrowwallet.frigate.electrum;
 
 import com.sparrowwallet.frigate.SubscriptionStatus;
-import com.sparrowwallet.frigate.index.TxEntry;
 
 import java.util.List;
 
-public record SilentPaymentsNotification(SilentPaymentsSubscription subscription, double progress, List<TxEntry> history, SubscriptionStatus status) {
+public record SilentPaymentsNotification(SilentPaymentsSubscription subscription, double progress, List<SilentPaymentsTxEntry> history, SubscriptionStatus status) {
 
 }

src/main/java/com/sparrowwallet/frigate/electrum/SilentPaymentsSubscription.java

@@ -1,5 +1,5 @@
 package com.sparrowwallet.frigate.electrum;
 
-public record SilentPaymentsSubscription(String address, int start_height) {
+public record SilentPaymentsSubscription(String address, Integer[] labels, int start_height) {
 
 }

src/main/java/com/sparrowwallet/frigate/electrum/SilentPaymentsTxEntry.java

@@ -0,0 +1,59 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import java.util.Objects;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class SilentPaymentsTxEntry implements Comparable<SilentPaymentsTxEntry> {
+    public int height;
+    public String tx_hash;
+    public String tweak_key;
+
+    public SilentPaymentsTxEntry() {
+    }
+
+    public SilentPaymentsTxEntry(int height, String tx_hash, String tweak_key) {
+        this.height = height;
+        this.tx_hash = tx_hash;
+        this.tweak_key = tweak_key;
+    }
+
+    public int getHeight() {
+        return height;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if(this == o) {
+            return true;
+        }
+        if(!(o instanceof SilentPaymentsTxEntry that)) {
+            return false;
+        }
+
+        return height == that.height && Objects.equals(tx_hash, that.tx_hash) && Objects.equals(tweak_key, that.tweak_key);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = height;
+        result = 31 * result + Objects.hashCode(tx_hash);
+        result = 31 * result + Objects.hashCode(tweak_key);
+        return result;
+    }
+
+    @Override
+    public int compareTo(SilentPaymentsTxEntry o) {
+        if(height <= 0 && o.height > 0) {
+            return 1;
+        }
+        if(height > 0 && o.height <= 0) {
+            return -1;
+        }
+        if(height != o.height) {
+            return height - o.height;
+        }
+        return tx_hash.compareTo(o.tx_hash);
+    }
+}

src/main/java/com/sparrowwallet/frigate/electrum/TransactionMerkle.java

@@ -0,0 +1,5 @@
+package com.sparrowwallet.frigate.electrum;
+
+import java.util.List;
+
+public record TransactionMerkle(int block_height, List<String> merkle, int pos) {}

src/main/java/com/sparrowwallet/frigate/electrum/TxEntry.java

@@ -1,4 +1,4 @@
-package com.sparrowwallet.frigate.index;
+package com.sparrowwallet.frigate.electrum;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.sparrowwallet.drongo.protocol.Transaction;

src/main/java/com/sparrowwallet/frigate/electrum/UnspentOutput.java

@@ -0,0 +1,3 @@
+package com.sparrowwallet.frigate.electrum;
+
+public record UnspentOutput(int height, int tx_pos, String tx_hash, long value) {}

src/main/java/com/sparrowwallet/frigate/electrum/VersionNotNegotiatedException.java

@@ -0,0 +1,6 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcError;
+
+@JsonRpcError(code=-1, message="server.version must be the first message")
+public class VersionNotNegotiatedException extends RuntimeException {}

src/main/java/com/sparrowwallet/frigate/index/AbstractDbManager.java

@@ -1,68 +1,20 @@
 package com.sparrowwallet.frigate.index;
 
-import com.sparrowwallet.frigate.io.Config;
-import com.sparrowwallet.frigate.io.Storage;
-import com.zaxxer.hikari.HikariConfig;
-import com.zaxxer.hikari.HikariDataSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
-import java.util.Properties;
 
 public abstract class AbstractDbManager implements DbManager {
     private static final Logger log = LoggerFactory.getLogger(AbstractDbManager.class);
 
+    public AbstractDbManager() {
+    }
+
     protected Connection createWriteConnection(String connectionUrl) throws SQLException {
         log.debug("Creating write connection");
         return DriverManager.getConnection(connectionUrl);
     }
-
-    protected HikariDataSource createReadDataSource(String connectionUrl, int maxPoolSize) {
-        HikariConfig config = new HikariConfig();
-        config.setJdbcUrl(connectionUrl);
-        config.setDriverClassName("org.duckdb.DuckDBDriver");
-        config.addDataSourceProperty("access_mode", "READ_ONLY");
-        config.addDataSourceProperty("allow_unsigned_extensions", "true");
-        config.addDataSourceProperty("jdbc_stream_results", "true");
-        config.addDataSourceProperty("scheduler_process_partial", "true");
-        config.setConnectionInitSql(buildReadConnectionInitSql());
-
-        config.setMaximumPoolSize(maxPoolSize);
-        config.setMinimumIdle(2);
-        config.setConnectionTimeout(30000);
-        config.setIdleTimeout(300000);
-        config.setMaxLifetime(900000);
-        config.setLeakDetectionThreshold(0);
-
-        config.setConnectionTestQuery("SELECT 1");
-        config.setValidationTimeout(5000);
-        config.setPoolName("DuckDB-ReadOnly-Pool");
-
-        log.debug("Creating read connection pool with max size: " + config.getMaximumPoolSize());
-        return new HikariDataSource(config);
-    }
-
-    private String buildReadConnectionInitSql() {
-        Properties duckDbProperties = new Properties();
-        duckDbProperties.setProperty("enable_progress_bar", "true");
-        duckDbProperties.setProperty("enable_progress_bar_print", "false");
-        if(Config.get().getDbThreads() != null) {
-            duckDbProperties.setProperty("threads", Config.get().getDbThreads().toString());
-        }
-
-        StringBuilder sql = new StringBuilder();
-        for(String propertyName : duckDbProperties.stringPropertyNames()) {
-            String value = duckDbProperties.getProperty(propertyName);
-            sql.append("SET ").append(propertyName).append(" = '").append(value).append("'; ");
-        }
-
-        File secp256k1ExtensionFile = Storage.getSecp256k1ExtensionFile();
-        sql.append("LOAD '").append(secp256k1ExtensionFile.getAbsolutePath()).append("'; ");
-
-        return sql.toString().trim();
-    }
 }

src/main/java/com/sparrowwallet/frigate/index/DuckDBReadPool.java

@@ -0,0 +1,98 @@
+package com.sparrowwallet.frigate.index;
+
+import com.sparrowwallet.frigate.io.Config;
+import com.sparrowwallet.frigate.io.Storage;
+import org.duckdb.DuckDBConnection;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.sql.*;
+import java.util.Properties;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+public class DuckDBReadPool {
+    private static final Logger log = LoggerFactory.getLogger(DuckDBReadPool.class);
+
+    private final DuckDBConnection masterConnection;
+    private final ArrayBlockingQueue<Connection> pool;
+    private final int maxSize;
+    private volatile boolean closed = false;
+
+    public DuckDBReadPool(String connectionUrl, int maxSize) throws SQLException {
+        this.maxSize = maxSize;
+
+        Properties props = new Properties();
+        props.setProperty("duckdb.read_only", "true");
+        props.setProperty("allow_unsigned_extensions", "true");
+        this.masterConnection = (DuckDBConnection)DriverManager.getConnection(connectionUrl, props);
+
+        try(Statement stmt = masterConnection.createStatement()) {
+            if(Config.get().getScan().getDbThreads() != null) {
+                stmt.execute("SET threads = '" + Config.get().getScan().getDbThreads() + "'");
+            }
+            if(Config.get().getScan().getMemoryLimit() != null) {
+                stmt.execute("SET memory_limit = '" + Config.get().getScan().getMemoryLimit() + "'");
+            }
+
+            File ufsecpExtensionFile = Storage.getUfsecpExtensionFile();
+            stmt.execute("LOAD '" + ufsecpExtensionFile.getAbsolutePath() + "'");
+            stmt.execute("SELECT ufsecp_set_cache_dir('" + Storage.getFrigateCacheDir().getAbsolutePath() + "')");
+        }
+
+        this.pool = new ArrayBlockingQueue<>(maxSize);
+        log.debug("DuckDB read pool created (max size: {})", maxSize);
+    }
+
+    public Connection getConnection() throws SQLException {
+        if(closed) {
+            throw new SQLException("Pool is closed");
+        }
+
+        Connection conn = pool.poll();
+        if(conn != null && !conn.isClosed()) {
+            return conn;
+        }
+
+        return masterConnection.duplicate();
+    }
+
+    public void releaseConnection(Connection conn) {
+        if(closed || conn == null) {
+            closeQuietly(conn);
+            return;
+        }
+
+        try {
+            if(conn.isClosed()) {
+                return;
+            }
+        } catch(SQLException e) {
+            return;
+        }
+
+        if(!pool.offer(conn)) {
+            closeQuietly(conn);
+        }
+    }
+
+    public void close() {
+        closed = true;
+        Connection conn;
+        while((conn = pool.poll()) != null) {
+            closeQuietly(conn);
+        }
+        closeQuietly(masterConnection);
+        log.debug("DuckDB read pool closed");
+    }
+
+    private static void closeQuietly(Connection conn) {
+        if(conn != null) {
+            try {
+                conn.close();
+            } catch(SQLException ignored) {
+            }
+        }
+    }
+}

src/main/java/com/sparrowwallet/frigate/index/HistoricalScanMetrics.java

@@ -0,0 +1,95 @@
+package com.sparrowwallet.frigate.index;
+
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicIntegerArray;
+
+/**
+ * Privacy-preserving aggregate counters for historical scan throughput.
+ */
+class HistoricalScanMetrics {
+    private static final int MIN_SAMPLES_PER_BUCKET = 10;
+    private static final int COUNT_ROUNDING = 10;
+
+    static final String[] RESULT_LABELS = {"0", "1-10", "11-100", "101-1000", "1001-10000", "10001+"};
+    static final String[] DURATION_LABELS = {"0-100ms", "100-500ms", "500ms-2s", "2-10s", "10-60s", "60s+"};
+
+    private final AtomicIntegerArray resultBuckets = new AtomicIntegerArray(RESULT_LABELS.length);
+    private final AtomicIntegerArray durationBuckets = new AtomicIntegerArray(DURATION_LABELS.length);
+
+    record Snapshot(int[] results, int[] durations) {}
+
+    void record(int resultCount, long durationMillis) {
+        resultBuckets.incrementAndGet(resultBucket(resultCount));
+        durationBuckets.incrementAndGet(durationBucket(durationMillis));
+    }
+
+    //a record() concurrent with snapshotAndReset() may have its two increments split across windows. The per-bucket drift is at
+    //most one sample per emission and never crosses subscription identity, so it is fine for an aggregate stat.
+    Snapshot snapshotAndReset() {
+        int[] results = new int[RESULT_LABELS.length];
+        int[] durations = new int[DURATION_LABELS.length];
+        for(int i = 0; i < results.length; i++) {
+            results[i] = resultBuckets.getAndSet(i, 0);
+        }
+        for(int i = 0; i < durations.length; i++) {
+            durations[i] = durationBuckets.getAndSet(i, 0);
+        }
+        return new Snapshot(results, durations);
+    }
+
+    Optional<String> format(Snapshot snapshot) {
+        String resultsStr = formatHist(snapshot.results(), RESULT_LABELS);
+        String durationsStr = formatHist(snapshot.durations(), DURATION_LABELS);
+        if(resultsStr.isEmpty() && durationsStr.isEmpty()) {
+            return Optional.empty();
+        }
+
+        StringBuilder sb = new StringBuilder("Aggregate SP scan stats (1h window):");
+        if(!resultsStr.isEmpty()) {
+            sb.append(" results [").append(resultsStr).append("]");
+        }
+        if(!durationsStr.isEmpty()) {
+            sb.append(" duration [").append(durationsStr).append("]");
+        }
+        return Optional.of(sb.toString());
+    }
+
+    static int resultBucket(int n) {
+        if(n <= 0) return 0;
+        if(n <= 10) return 1;
+        if(n <= 100) return 2;
+        if(n <= 1000) return 3;
+        if(n <= 10000) return 4;
+        return 5;
+    }
+
+    static int durationBucket(long ms) {
+        if(ms < 100) return 0;
+        if(ms < 500) return 1;
+        if(ms < 2000) return 2;
+        if(ms < 10000) return 3;
+        if(ms < 60000) return 4;
+        return 5;
+    }
+
+    private static String formatHist(int[] counts, String[] labels) {
+        StringBuilder sb = new StringBuilder();
+        for(int i = 0; i < counts.length; i++) {
+            int rounded = roundCount(counts[i]);
+            if(rounded > 0) {
+                if(sb.length() > 0) {
+                    sb.append(", ");
+                }
+                sb.append(labels[i]).append(":").append(rounded);
+            }
+        }
+        return sb.toString();
+    }
+
+    private static int roundCount(int raw) {
+        if(raw < MIN_SAMPLES_PER_BUCKET) {
+            return 0;
+        }
+        return ((raw + COUNT_ROUNDING / 2) / COUNT_ROUNDING) * COUNT_ROUNDING;
+    }
+}

src/main/java/com/sparrowwallet/frigate/index/Index.java

@@ -2,46 +2,73 @@ package com.sparrowwallet.frigate.index;
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.sparrowwallet.drongo.Utils;
+import com.sparrowwallet.drongo.crypto.ECKey;
 import com.sparrowwallet.drongo.protocol.*;
 import com.sparrowwallet.drongo.silentpayments.SilentPaymentScanAddress;
+import com.sparrowwallet.drongo.silentpayments.SilentPaymentUtils;
 import com.sparrowwallet.drongo.wallet.BlockTransaction;
 import com.sparrowwallet.frigate.ConfigurationException;
 import com.sparrowwallet.frigate.Frigate;
 import com.sparrowwallet.frigate.SubscriptionStatus;
 import com.sparrowwallet.frigate.electrum.SilentPaymentsNotification;
 import com.sparrowwallet.frigate.electrum.SilentPaymentsSubscription;
+import com.sparrowwallet.frigate.electrum.SilentPaymentsTxEntry;
+import com.sparrowwallet.frigate.io.ComputeBackend;
 import com.sparrowwallet.frigate.io.Config;
 import com.sparrowwallet.frigate.io.Storage;
+import org.duckdb.DuckDBAppender;
+import org.duckdb.DuckDBConnection;
 import org.duckdb.DuckDBPreparedStatement;
-import org.duckdb.QueryProgress;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.lang.ref.WeakReference;
+import java.math.BigInteger;
 import java.sql.*;
 import java.util.*;
 import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BooleanSupplier;
 import java.util.stream.Collectors;
 
 public class Index {
     private static final Logger log = LoggerFactory.getLogger(Index.class);
     public static final String DEFAULT_DB_FILENAME = "frigate.duckdb";
     private static final String TWEAK_TABLE = "tweak";
+    private static final String INDEXED_BLOCK_TABLE = "indexed_block";
     public static final int HISTORY_PAGE_SIZE = 100;
 
-    private final DbManager dbManager;
-    private int lastBlockIndexed = -1;
+    private static final String AUDIT_SCAN_KEY_ENV = "FRIGATE_AUDIT_SCAN_KEY";
+    private static final String AUDIT_SPEND_KEY_ENV = "FRIGATE_AUDIT_SPEND_KEY";
 
-    public Index(int startHeight, boolean inMemory) {
-        lastBlockIndexed = Math.max(lastBlockIndexed, startHeight - 1);
+    private final DbManager dbManager;
+    private final AtomicInteger lastBlockIndexed = new AtomicInteger(-1);
+    private final int batchSize;
+    private final ECKey auditScanKey;
+    private final ECKey auditSpendKey;
+    private volatile boolean steadyState = false;
+
+    public Index(int startHeight, boolean inMemory, int batchSize) {
+        lastBlockIndexed.accumulateAndGet(startHeight - 1, Math::max);
+        this.batchSize = batchSize;
+
+        String scanKeyHex = System.getenv(AUDIT_SCAN_KEY_ENV);
+        String spendKeyHex = System.getenv(AUDIT_SPEND_KEY_ENV);
+        if(scanKeyHex != null && spendKeyHex != null) {
+            this.auditScanKey = ECKey.fromPrivate(Utils.hexToBytes(scanKeyHex));
+            this.auditSpendKey = ECKey.fromPublicOnly(Utils.hexToBytes(spendKeyHex));
+            log.warn("Scan audit mode enabled — output prefixes will be computed for the provided wallet keys");
+        } else {
+            this.auditScanKey = null;
+            this.auditSpendKey = null;
+        }
 
         if(inMemory) {
             dbManager = new MemoryDbManager();
         } else {
-            String dbUrl = Config.get().getDbUrl();
-            List<String> readDbUrls = Config.get().getReadDbUrls();
+            String dbUrl = Config.get().getDatabase().getUrl();
+            List<String> readDbUrls = Config.get().getDatabase().getReadUrls();
             if(dbUrl != null && readDbUrls != null && !readDbUrls.isEmpty()) {
                 dbManager = new ScalingDbManager(dbUrl, readDbUrls);
             } else if(dbUrl == null) {
@@ -55,11 +82,89 @@ public class Index {
         try {
             dbManager.executeWrite(connection -> {
                 try(Statement stmt = connection.createStatement()) {
-                    return stmt.execute("CREATE TABLE IF NOT EXISTS " + TWEAK_TABLE + " (txid BLOB NOT NULL, height INTEGER NOT NULL, tweak_key BLOB NOT NULL, outputs BIGINT[])");
+                    stmt.execute("CREATE TABLE IF NOT EXISTS " + TWEAK_TABLE + " (txid BLOB NOT NULL, height INTEGER NOT NULL, tweak_key BLOB NOT NULL, outputs BIGINT[])");
+                    stmt.execute("CREATE TABLE IF NOT EXISTS " + INDEXED_BLOCK_TABLE + " (height INTEGER NOT NULL, block_hash BLOB NOT NULL, singleton BOOLEAN PRIMARY KEY DEFAULT true CHECK (singleton))");
+                    return true;
+                }
+            });
+            seedIndexedBlockIfEmpty();
+        } catch(Exception e) {
+            throw new ConfigurationException("Error initialising index", e);
+        }
+
+        if(!inMemory) {
+            checkGpuBackend();
+        }
+    }
+
+    private void seedIndexedBlockIfEmpty() throws SQLException, InterruptedException {
+        dbManager.executeWrite(connection -> {
+            try(Statement stmt = connection.createStatement();
+                ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM " + INDEXED_BLOCK_TABLE)) {
+                if(rs.next() && rs.getInt(1) == 0) {
+                    try(ResultSet maxRs = stmt.executeQuery("SELECT MAX(height) FROM " + TWEAK_TABLE)) {
+                        if(maxRs.next() && maxRs.getObject(1) != null) {
+                            int maxHeight = maxRs.getInt(1);
+                            if(maxHeight > 0) {
+                                try(PreparedStatement ins = connection.prepareStatement("INSERT INTO " + INDEXED_BLOCK_TABLE + " (height, block_hash) VALUES (?, ?)")) {
+                                    ins.setInt(1, maxHeight);
+                                    ins.setBytes(2, new byte[0]);
+                                    ins.executeUpdate();
+                                }
+                            }
+                        }
+                    }
+                }
+
+                return true;
+            }
+        });
+    }
+
+    private void checkGpuBackend() {
+        ComputeBackend computeBackend = Config.get().getScan().getComputeBackendEnum();
+        if(computeBackend == ComputeBackend.CPU) {
+            return;
+        }
+
+        try {
+            String backend = dbManager.executeRead(connection -> {
+                try(Statement stmt = connection.createStatement(); ResultSet rs = stmt.executeQuery("SELECT ufsecp_backend()")) {
+                    return rs.next() ? rs.getString(1) : "unknown";
+                }
+            });
+
+            if(backend.startsWith("cpu")) {
+                if(computeBackend == ComputeBackend.GPU) {
+                    throw new ConfigurationException("No GPU detected, but \"computeBackend\" is set to \"GPU\". Set to \"AUTO\" or \"CPU\", or install a supported GPU.");
+                }
+                log.info("Using CPU backend for scanning (no GPU detected)");
+            } else {
+                log.info("Using {} backend for scanning", backend);
+            }
+        } catch(Exception e) {
+            log.warn("Could not detect GPU backend", e);
+        }
+    }
+
+    private double pollScanProgress(byte[] scanKeyBytes) {
+        try {
+            return dbManager.executeRead(progressConnection -> {
+                try(PreparedStatement progressStmt = progressConnection.prepareStatement("SELECT ufsecp_progress(?)")) {
+                    progressStmt.setBytes(1, scanKeyBytes);
+                    ResultSet rs = progressStmt.executeQuery();
+                    if(rs.next()) {
+                        double pct = rs.getDouble(1);
+                        if(pct < 0.0d) {
+                            return 0.0d;
+                        }
+                        return Math.min(pct / 100.0d, 1.0d);
+                    }
+                    return 0.0d;
                 }
             });
         } catch(Exception e) {
-            throw new ConfigurationException("Error initialising index", e);
+            return 0.0d;
         }
     }
 
@@ -67,71 +172,149 @@ public class Index {
         dbManager.close();
     }
 
-    public int getLastBlockIndexed() {
-        try {
-            return dbManager.executeRead(connection -> {
-                try(PreparedStatement statement = connection.prepareStatement("SELECT MAX(height) from " + TWEAK_TABLE)) {
-                    ResultSet resultSet = statement.executeQuery();
-                    return resultSet.next() ? Math.max(lastBlockIndexed, resultSet.getInt(1)) : lastBlockIndexed;
-                }
-            });
-        } catch(Exception e) {
-            log.error("Error getting last block indexed", e);
-            return lastBlockIndexed;
-        }
+    public void setSteadyState(boolean steadyState) {
+        this.steadyState = steadyState;
     }
 
-    public void addToIndex(Map<BlockTransaction, byte[]> transactions) {
+    public void repairOrphanTweaks() {
         if(dbManager.isShutdown()) {
             return;
         }
 
-        int fromBlockHeight = lastBlockIndexed;
         try {
-            lastBlockIndexed = dbManager.executeWrite(connection -> {
-                try(PreparedStatement statement = connection.prepareStatement("INSERT INTO " + TWEAK_TABLE + " VALUES (?, ?, ?, ?)")) {
-                    int blockHeight = -1;
-
-                    for(BlockTransaction blkTx : transactions.keySet()) {
-                        statement.setBytes(1, blkTx.getTransaction().getTxId().getBytes());
-                        statement.setInt(2, blkTx.getHeight());
-                        statement.setObject(3, transactions.get(blkTx));
-
-                        List<TransactionOutput> outputs = blkTx.getTransaction().getOutputs();
-                        List<Long> hashPrefixes = new ArrayList<>();
-                        for(TransactionOutput output : outputs) {
-                            if(ScriptType.P2TR.isScriptType(output.getScript())) {
-                                long hashPrefix = getHashPrefix(ScriptType.P2TR.getPublicKeyFromScript(output.getScript()).getPubKey(), 1);
-                                hashPrefixes.add(hashPrefix);
-                            }
-                        }
-                        statement.setArray(4, connection.createArrayOf("BIGINT", hashPrefixes.toArray()));
-                        statement.addBatch();
-
-                        blockHeight = Math.max(blockHeight, blkTx.getHeight());
-                    }
-
-                    statement.executeBatch();
-                    if(fromBlockHeight < 0) {
-                        log.info("Indexed " + transactions.size() + " mempool transactions");
-                    } else if(blockHeight > 0) {
-                        log.info("Indexed " + transactions.size() + " transactions to block height " + blockHeight);
-                    }
-
-                    return blockHeight;
+            int deleted = dbManager.executeWrite(connection -> {
+                try(PreparedStatement ps = connection.prepareStatement("DELETE FROM " + TWEAK_TABLE + " WHERE height > (SELECT height FROM " + INDEXED_BLOCK_TABLE + ")")) {
+                    return ps.executeUpdate();
                 }
             });
+            if(deleted > 0) {
+                log.info("Removed {} orphan tweak rows above the indexed-block marker (interrupted shutdown recovery)", deleted);
+            }
+        } catch(Exception e) {
+            log.error("Error repairing orphan tweak rows", e);
+        }
+    }
 
-            if(lastBlockIndexed <= 0) {
+    public int getLastBlockIndexed() {
+        try {
+            return dbManager.executeRead(connection -> {
+                try(PreparedStatement statement = connection.prepareStatement("SELECT height FROM " + INDEXED_BLOCK_TABLE)) {
+                    ResultSet resultSet = statement.executeQuery();
+                    return resultSet.next() ? Math.max(lastBlockIndexed.get(), resultSet.getInt(1)) : lastBlockIndexed.get();
+                }
+            });
+        } catch(Exception e) {
+            log.error("Error getting last block indexed", e);
+            return lastBlockIndexed.get();
+        }
+    }
+
+    public byte[] getLastBlockHash() {
+        try {
+            return dbManager.executeRead(connection -> {
+                try(PreparedStatement statement = connection.prepareStatement("SELECT block_hash FROM " + INDEXED_BLOCK_TABLE)) {
+                    ResultSet resultSet = statement.executeQuery();
+                    if(!resultSet.next()) {
+                        return null;
+                    }
+                    byte[] hash = resultSet.getBytes(1);
+                    return (hash == null || hash.length == 0) ? null : hash;
+                }
+            });
+        } catch(Exception e) {
+            log.error("Error getting last block hash", e);
+            return null;
+        }
+    }
+
+    public void addToIndex(int height, byte[] blockHash, Map<BlockTransaction, byte[]> transactions) {
+        if(dbManager.isShutdown()) {
+            return;
+        }
+
+        int fromBlockHeight = lastBlockIndexed.get();
+        try {
+            int newLastBlockIndexed = dbManager.executeWrite(connection -> {
+                if(!transactions.isEmpty()) {
+                    DuckDBConnection duckDBConnection = (DuckDBConnection)connection;
+                    try(DuckDBAppender appender = duckDBConnection.createAppender(DuckDBConnection.DEFAULT_SCHEMA, TWEAK_TABLE)) {
+                        int blockHeight = -1;
+
+                        for(BlockTransaction blkTx : transactions.keySet()) {
+                            appender.beginRow();
+                            appender.append(blkTx.getTransaction().getTxId().getBytes());
+                            appender.append(blkTx.getHeight());
+                            appender.append(transactions.get(blkTx));
+
+                            List<Long> hashPrefixes = new ArrayList<>();
+                            if(auditScanKey != null) {
+                                long hashPrefix = getAuditHashPrefix(transactions, blkTx);
+                                hashPrefixes.add(hashPrefix);
+                            } else {
+                                List<TransactionOutput> outputs = blkTx.getTransaction().getOutputs();
+                                for(TransactionOutput output : outputs) {
+                                    if(ScriptType.P2TR.isScriptType(output.getScript())) {
+                                        long hashPrefix = getHashPrefix(ScriptType.P2TR.getPublicKeyFromScript(output.getScript()).getPubKey(), 1);
+                                        hashPrefixes.add(hashPrefix);
+                                    }
+                                }
+                            }
+                            appender.append(hashPrefixes.stream().mapToLong(Long::longValue).toArray());
+                            appender.endRow();
+
+                            blockHeight = Math.max(blockHeight, blkTx.getHeight());
+                        }
+
+                        if(blockHeight <= 0 && lastBlockIndexed.get() < 0) {
+                            log.info("Indexed " + transactions.size() + " mempool transactions");
+                        } else if(blockHeight > 0) {
+                            String msg = "Indexed " + transactions.size() + " transactions to block height " + blockHeight;
+                            if(steadyState) {
+                                log.info(msg);
+                            } else {
+                                log.debug(msg);
+                            }
+                        }
+                    }
+                }
+
+                if(height > 0 && blockHash != null) {
+                    try(PreparedStatement ps = connection.prepareStatement("INSERT INTO " + INDEXED_BLOCK_TABLE + " (height, block_hash) VALUES (?, ?) " +
+                            "ON CONFLICT (singleton) DO UPDATE SET height = excluded.height, block_hash = excluded.block_hash")) {
+                        ps.setInt(1, height);
+                        ps.setBytes(2, blockHash);
+                        ps.executeUpdate();
+                    }
+                }
+
+                return height;
+            });
+            lastBlockIndexed.set(newLastBlockIndexed);
+
+            if(transactions.isEmpty()) {
+                //empty block: marker advanced, but nothing to notify on
+            } else if(newLastBlockIndexed <= 0) {
                 Frigate.getEventBus().post(new SilentPaymentsMempoolIndexAdded(transactions.keySet().stream().map(blkTx -> blkTx.getTransaction().getTxId()).collect(Collectors.toSet())));
             } else {
-                Frigate.getEventBus().post(new SilentPaymentsBlocksIndexUpdate(fromBlockHeight + 1, lastBlockIndexed, transactions.size()));
+                Frigate.getEventBus().post(new SilentPaymentsBlocksIndexUpdate(fromBlockHeight + 1, newLastBlockIndexed, transactions.size()));
             }
         } catch(Exception e) {
             log.error("Error adding to index", e);
         }
     }
 
+    private long getAuditHashPrefix(Map<BlockTransaction, byte[]> transactions, BlockTransaction blkTx) {
+        byte[] tweakKeyBytes = transactions.get(blkTx);
+        ECKey tweakKey = ECKey.fromPublicOnly(compressRawKey(tweakKeyBytes));
+        ECKey sharedSecret = tweakKey.multiply(auditScanKey.getPrivKey(), true);
+        byte[] ser37 = new byte[37];
+        System.arraycopy(sharedSecret.getPubKey(true), 0, ser37, 0, 33);
+        byte[] t_k = Utils.taggedHash("BIP0352/SharedSecret", ser37);
+        ECKey tkG = ECKey.fromPublicOnly(ECKey.publicKeyFromPrivate(new BigInteger(1, t_k), true));
+        ECKey P0 = auditSpendKey.add(tkG, true);
+        return getHashPrefix(P0.getPubKeyXCoord(), 0);
+    }
+
     public void removeFromIndex(int startHeight) {
         if(dbManager.isShutdown()) {
             return;
@@ -139,11 +322,30 @@ public class Index {
 
         try {
             dbManager.executeWrite(connection -> {
-                try(PreparedStatement statement = connection.prepareStatement("DELETE FROM " + TWEAK_TABLE + " WHERE height >= ?")) {
-                    statement.setInt(1, startHeight);
-                    return statement.execute();
+                boolean prevAutoCommit = connection.getAutoCommit();
+                connection.setAutoCommit(false);
+                try {
+                    try(PreparedStatement deleteTweak = connection.prepareStatement("DELETE FROM " + TWEAK_TABLE + " WHERE height >= ?")) {
+                        deleteTweak.setInt(1, startHeight);
+                        deleteTweak.execute();
+                    }
+                    //hash applied to the original (higher) marker height — clear it so the startup hash check is a no-op until the first re-indexed block writes a real hash
+                    try(PreparedStatement updateMarker = connection.prepareStatement("INSERT INTO " + INDEXED_BLOCK_TABLE + " (height, block_hash) VALUES (?, ?) " +
+                            "ON CONFLICT (singleton) DO UPDATE SET height = excluded.height, block_hash = excluded.block_hash")) {
+                        updateMarker.setInt(1, Math.max(0, startHeight - 1));
+                        updateMarker.setBytes(2, new byte[0]);
+                        updateMarker.executeUpdate();
+                    }
+                    connection.commit();
+                    return true;
+                } catch(SQLException e) {
+                    connection.rollback();
+                    throw e;
+                } finally {
+                    connection.setAutoCommit(prevAutoCommit);
                 }
             });
+            lastBlockIndexed.accumulateAndGet(startHeight - 1, Math::min);
         } catch(Exception e) {
             log.error("Error removing from index", e);
         }
@@ -173,88 +375,72 @@ public class Index {
         }
     }
 
-    public List<TxEntry> getHistoryAsync(SilentPaymentScanAddress scanAddress, SilentPaymentsSubscription subscription, Integer startHeight, Integer endHeight, WeakReference<SubscriptionStatus> subscriptionStatusRef) {
-        ConcurrentLinkedQueue<TxEntry> queue = new ConcurrentLinkedQueue<>();
-        AtomicLong rowsProcessedStart = new AtomicLong(0L);
+    public List<SilentPaymentsTxEntry> getHistoryAsync(SilentPaymentScanAddress scanAddress, SilentPaymentsSubscription subscription, Integer startHeight, Integer endHeight, Set<Sha256Hash> mempoolTxids, WeakReference<SubscriptionStatus> subscriptionStatusRef, BooleanSupplier cancelled, boolean isHistorical) {
+        if(mempoolTxids != null && mempoolTxids.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        ConcurrentLinkedQueue<SilentPaymentsTxEntry> queue = new ConcurrentLinkedQueue<>();
+        byte[] scanKeyBytes = Utils.reverseBytes(scanAddress.getScanKey().getPrivKeyBytes());
 
         try {
             dbManager.executeRead(connection -> {
-                String sql = "SELECT txid, height FROM " + TWEAK_TABLE +
-                        " WHERE list_contains(outputs, hash_prefix_to_int(secp256k1_ec_pubkey_combine([?, secp256k1_ec_pubkey_create(secp256k1_tagged_sha256('BIP0352/SharedSecret', secp256k1_ec_pubkey_tweak_mul(tweak_key, ?) || int_to_big_endian(0)))]), 1))";
-
-                if(startHeight != null) {
-                    sql += " AND height >= ?";
-                }
-                if(endHeight != null) {
-                    sql += " AND height <= ?";
-                }
+                String sql = getSql(subscription, startHeight, endHeight, mempoolTxids, isHistorical);
 
                 try(DuckDBPreparedStatement statement = connection.prepareStatement(sql).unwrap(DuckDBPreparedStatement.class)) {
-                    if(isUnsubscribed(scanAddress, subscriptionStatusRef)) {
+                    if(isUnsubscribed(scanAddress, subscriptionStatusRef) || cancelled.getAsBoolean()) {
                         return false;
                     }
 
-                    statement.setBytes(1, scanAddress.getSpendKey().getPubKey());
-                    statement.setBytes(2, scanAddress.getScanKey().getPrivKeyBytes());
-                    if(startHeight != null) {
-                        statement.setInt(3, startHeight);
-                    }
-                    if(endHeight != null) {
-                        statement.setInt(startHeight == null ? 3 : 4, endHeight);
-                    }
-                    statement.setFetchSize(1);
+                    Long totalRows = isHistorical ? getInputRowCount(connection, startHeight, endHeight) : null;
+                    bindParameters(statement, scanAddress, subscription, startHeight, endHeight, mempoolTxids, isHistorical, totalRows);
 
-                    try(ScheduledThreadPoolExecutor queryProgressExecutor = new ScheduledThreadPoolExecutor(1, r -> {
-                        ThreadFactory namedThreadFactory = new ThreadFactoryBuilder().setNameFormat("IndexQueryProgress-%d").build();
-                        Thread t = namedThreadFactory.newThread(r);
-                        t.setDaemon(true);
-                        return t;
-                    })) {
-                        queryProgressExecutor.scheduleAtFixedRate(() -> {
-                            try {
-                                if(dbManager.isShutdown() || isUnsubscribed(scanAddress, subscriptionStatusRef)) {
-                                    statement.cancel();
-                                    queryProgressExecutor.shutdownNow();
-                                    return;
-                                }
-
-                                QueryProgress queryProgress = statement.getQueryProgress();
-                                if(queryProgress.getRowsProcessed() == queryProgress.getTotalRowsToProcess()) {
-                                    return;
-                                }
-
-                                double progress = 0.0d;
-                                if(rowsProcessedStart.get() == 0L && queryProgress.getRowsProcessed() > 0) {
-                                    rowsProcessedStart.set(queryProgress.getRowsProcessed());
-                                }
-                                if(rowsProcessedStart.get() > 0L) {
-                                    progress = (queryProgress.getRowsProcessed() - rowsProcessedStart.get()) / (double)(queryProgress.getTotalRowsToProcess() - rowsProcessedStart.get());
-                                }
-
-                                List<TxEntry> history = new ArrayList<>();
-                                TxEntry entry;
-                                while((entry = queue.poll()) != null) {
-                                    history.add(entry);
-                                    if(history.size() >= HISTORY_PAGE_SIZE) {
-                                        Frigate.getEventBus().post(new SilentPaymentsNotification(subscription, progress, new ArrayList<>(history), subscriptionStatusRef.get()));
-                                        history.clear();
+                    if(isHistorical) {
+                        try(ScheduledThreadPoolExecutor queryProgressExecutor = new ScheduledThreadPoolExecutor(1, r -> {
+                            ThreadFactory namedThreadFactory = new ThreadFactoryBuilder().setNameFormat("IndexQueryProgress-%d").build();
+                            Thread t = namedThreadFactory.newThread(r);
+                            t.setDaemon(true);
+                            return t;
+                        })) {
+                            queryProgressExecutor.scheduleAtFixedRate(() -> {
+                                try {
+                                    if(queryProgressExecutor.isShutdown() || dbManager.isShutdown() || isUnsubscribed(scanAddress, subscriptionStatusRef) || cancelled.getAsBoolean()) {
+                                        statement.cancel();
+                                        queryProgressExecutor.shutdownNow();
+                                        return;
+                                    }
+
+                                    double progress = pollScanProgress(scanKeyBytes);
+
+                                    if(queryProgressExecutor.isShutdown()) {
+                                        return;
+                                    }
+
+                                    List<SilentPaymentsTxEntry> history = new ArrayList<>();
+                                    SilentPaymentsTxEntry entry;
+                                    while((entry = queue.poll()) != null) {
+                                        history.add(entry);
+                                        if(history.size() >= HISTORY_PAGE_SIZE) {
+                                            Frigate.getEventBus().post(new SilentPaymentsNotification(subscription, progress, new ArrayList<>(history), subscriptionStatusRef.get()));
+                                            history.clear();
+                                        }
                                     }
-                                }
-                                if(!history.isEmpty() || queryProgressExecutor.getTaskCount() % 5 == 0) {
                                     Frigate.getEventBus().post(new SilentPaymentsNotification(subscription, progress, new ArrayList<>(history), subscriptionStatusRef.get()));
                                     history.clear();
+                                } catch(Exception e) {
+                                    log.error("Error getting query progress", e);
                                 }
-                            } catch(SQLException e) {
-                                log.error("Error getting query progress", e);
-                            }
-                        }, 1, 1, TimeUnit.SECONDS);
+                            }, 5, 5, TimeUnit.SECONDS);
 
-                        ResultSet resultSet = statement.executeQuery();
-                        while(resultSet.next()) {
-                            byte[] txid = resultSet.getBytes(1);
-                            int height = resultSet.getInt(2);
-                            queue.offer(new TxEntry(height, 0, Utils.bytesToHex(txid)));
+                            try {
+                                drainResultSet(statement.executeQuery(), queue);
+                            } finally {
+                                //interrupt any parked progress poll - the implicit close() doesn't, which deadlocks against a queued writer
+                                queryProgressExecutor.shutdownNow();
+                            }
                         }
+                    } else {
+                        drainResultSet(statement.executeQuery(), queue);
                     }
                 }
 
@@ -272,12 +458,12 @@ public class Index {
             return Collections.emptyList();
         }
 
-        if(isUnsubscribed(scanAddress, subscriptionStatusRef)) {
+        if(isUnsubscribed(scanAddress, subscriptionStatusRef) || cancelled.getAsBoolean()) {
             return Collections.emptyList();
         }
 
-        List<TxEntry> history = new ArrayList<>();
-        TxEntry entry;
+        List<SilentPaymentsTxEntry> history = new ArrayList<>();
+        SilentPaymentsTxEntry entry;
         while((entry = queue.poll()) != null) {
             history.add(entry);
         }
@@ -285,6 +471,133 @@ public class Index {
         return history;
     }
 
+    private void drainResultSet(ResultSet resultSet, ConcurrentLinkedQueue<SilentPaymentsTxEntry> queue) throws SQLException {
+        while(resultSet.next()) {
+            byte[] txid = resultSet.getBytes(1);
+            byte[] tweak_key = compressRawKey(resultSet.getBytes(2));
+            int height = resultSet.getInt(3);
+            queue.offer(new SilentPaymentsTxEntry(height, Utils.bytesToHex(txid), Utils.bytesToHex(tweak_key)));
+        }
+    }
+
+    private String getSql(SilentPaymentsSubscription subscription, Integer startHeight, Integer endHeight, Set<Sha256Hash> mempoolTxids, boolean isHistorical) {
+        String labelsStr = "[" + String.join(", ", Collections.nCopies(subscription.labels().length, "?")) + "]";
+
+        String sql = "SELECT txid, tweak_key, height FROM ufsecp_scan((SELECT txid, height, tweak_key, outputs FROM " + TWEAK_TABLE
+                + buildWhereClause(startHeight, endHeight, mempoolTxids)
+                + "), ?, ?, " + labelsStr + ", batch_size := ?";
+
+        ComputeBackend backend = resolveBackend(isHistorical);
+        if(backend != ComputeBackend.AUTO) {
+            sql += ", backend := ?";
+        }
+
+        if(isHistorical) {
+            sql += ", total_rows := ?";
+        }
+
+        sql += ") ORDER BY height";
+
+        return sql;
+    }
+
+    private void bindParameters(DuckDBPreparedStatement statement, SilentPaymentScanAddress scanAddress, SilentPaymentsSubscription subscription, Integer startHeight, Integer endHeight, Set<Sha256Hash> mempoolTxids, boolean isHistorical, Long totalRows) throws SQLException {
+        int index = bindTweakHeightFilter(statement, 1, startHeight, endHeight);
+        index = bindTweakTxidsFilter(statement, index, mempoolTxids);
+        statement.setBytes(index++, Utils.reverseBytes(scanAddress.getScanKey().getPrivKeyBytes()));
+        statement.setBytes(index++, SilentPaymentUtils.getSecp256k1PubKey(scanAddress.getSpendKey()));
+        for(Integer label : subscription.labels()) {
+            statement.setBytes(index++, SilentPaymentUtils.getSecp256k1PubKey(scanAddress.getLabelledTweakKey(label)));
+        }
+        statement.setInt(index++, batchSize);
+
+        ComputeBackend backend = resolveBackend(isHistorical);
+        if(backend != ComputeBackend.AUTO) {
+            statement.setString(index++, backend.toSqlValue());
+        }
+
+        if(totalRows != null) {
+            statement.setLong(index, totalRows);
+        }
+    }
+
+    private static String buildWhereClause(Integer startHeight, Integer endHeight, Set<Sha256Hash> mempoolTxids) {
+        String heightClause = tweakHeightFilter(startHeight, endHeight);
+        String txidsClause = tweakTxidsFilter(mempoolTxids);
+        if(heightClause.isEmpty() && txidsClause.isEmpty()) {
+            return "";
+        }
+        if(heightClause.isEmpty()) {
+            return " WHERE " + txidsClause;
+        }
+        if(txidsClause.isEmpty()) {
+            return " WHERE " + heightClause;
+        }
+        return " WHERE " + heightClause + " AND " + txidsClause;
+    }
+
+    private static String tweakHeightFilter(Integer startHeight, Integer endHeight) {
+        if(startHeight == null && endHeight == null) {
+            return "";
+        }
+        StringBuilder sql = new StringBuilder();
+        if(startHeight != null) {
+            sql.append("height >= ?");
+            if(endHeight != null) {
+                sql.append(" AND ");
+            }
+        }
+        if(endHeight != null) {
+            sql.append("height <= ?");
+        }
+        return sql.toString();
+    }
+
+    private static String tweakTxidsFilter(Set<Sha256Hash> mempoolTxids) {
+        if(mempoolTxids == null) {
+            return "";
+        }
+        return "txid IN (SELECT unnest(?))";
+    }
+
+    private static int bindTweakHeightFilter(PreparedStatement statement, int startIndex, Integer startHeight, Integer endHeight) throws SQLException {
+        int idx = startIndex;
+        if(startHeight != null) {
+            statement.setInt(idx++, startHeight);
+        }
+        if(endHeight != null) {
+            statement.setInt(idx++, endHeight);
+        }
+        return idx;
+    }
+
+    private static int bindTweakTxidsFilter(PreparedStatement statement, int startIndex, Set<Sha256Hash> mempoolTxids) throws SQLException {
+        if(mempoolTxids == null) {
+            return startIndex;
+        }
+        Object[] arr = mempoolTxids.stream().map(Sha256Hash::getBytes).toArray();
+        Array array = statement.getConnection().createArrayOf("BLOB", arr);
+        statement.setArray(startIndex, array);
+        return startIndex + 1;
+    }
+
+    private long getInputRowCount(Connection connection, Integer startHeight, Integer endHeight) throws SQLException {
+        String sql = "SELECT COUNT(*) FROM " + TWEAK_TABLE + buildWhereClause(startHeight, endHeight, null);
+        try(PreparedStatement countStmt = connection.prepareStatement(sql)) {
+            bindTweakHeightFilter(countStmt, 1, startHeight, endHeight);
+            ResultSet rs = countStmt.executeQuery();
+            return rs.next() ? rs.getLong(1) : 0L;
+        }
+    }
+
+    private static ComputeBackend resolveBackend(boolean isHistorical) {
+        if(!isHistorical) {
+            return ComputeBackend.CPU;
+        }
+
+        return Config.get().getScan().getComputeBackendEnum();
+    }
+
     private static boolean isUnsubscribed(SilentPaymentScanAddress scanAddress, WeakReference<SubscriptionStatus> subscriptionStatusRef) {
         SubscriptionStatus status = subscriptionStatusRef.get();
         return status == null || !status.isConnected() || !status.isSilentPaymentsAddressSubscribed(scanAddress.toString());
@@ -302,4 +615,14 @@ public class Index {
         }
         return result;
     }
+
+    public static byte[] compressRawKey(byte[] rawUncompressed) {
+        byte[] uncompressed = new byte[64];
+        System.arraycopy(rawUncompressed, 0, uncompressed, 32, 32);
+        System.arraycopy(rawUncompressed, 32, uncompressed, 0, 32);
+        uncompressed = Utils.reverseBytes(uncompressed);
+
+        ECKey ecKey = ECKey.fromPublicOnly(Utils.concat(new byte[] {0x04}, uncompressed));
+        return ecKey.getPubKey(true);
+    }
 }

src/main/java/com/sparrowwallet/frigate/index/IndexQuerier.java

@@ -5,25 +5,52 @@ import com.sparrowwallet.drongo.protocol.Sha256Hash;
 import com.sparrowwallet.drongo.silentpayments.SilentPaymentScanAddress;
 import com.sparrowwallet.frigate.Frigate;
 import com.sparrowwallet.frigate.SubscriptionStatus;
+import com.sparrowwallet.frigate.electrum.SilentPaymentAddressSubscription;
 import com.sparrowwallet.frigate.electrum.SilentPaymentsNotification;
 import com.sparrowwallet.frigate.electrum.SilentPaymentsSubscription;
+import com.sparrowwallet.frigate.electrum.SilentPaymentsTxEntry;
+import com.sparrowwallet.frigate.io.Config;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.lang.ref.WeakReference;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
 
 public class IndexQuerier {
+    private static final Logger log = LoggerFactory.getLogger(IndexQuerier.class);
+
     public static final double PROGRESS_COMPLETE = 1.0d;
 
     private final Index blocksIndex;
     private final Index mempoolIndex;
+    private final HistoricalScanMetrics metrics;
+    private final ScheduledExecutorService metricsExecutor;
 
     public IndexQuerier(Index blocksIndex, Index mempoolIndex) {
         this.blocksIndex = blocksIndex;
         this.mempoolIndex = mempoolIndex;
+
+        if(Config.get().getScan().isMetricsEnabled()) {
+            this.metrics = new HistoricalScanMetrics();
+            this.metricsExecutor = Executors.newSingleThreadScheduledExecutor(r -> {
+                Thread t = new ThreadFactoryBuilder().setNameFormat("IndexQueryMetrics-%d").build().newThread(r);
+                t.setDaemon(true);
+                return t;
+            });
+            this.metricsExecutor.scheduleAtFixedRate(this::emitMetrics, 1, 1, TimeUnit.HOURS);
+        } else {
+            this.metrics = null;
+            this.metricsExecutor = null;
+        }
     }
 
     private final ExecutorService queryPool = Executors.newFixedThreadPool(10, r -> {
@@ -33,36 +60,65 @@ public class IndexQuerier {
         return t;
     });
 
-    public void startHistoryScan(SilentPaymentScanAddress scanAddress, Integer startHeight, Integer endHeight, WeakReference<SubscriptionStatus> subscriptionStatusRef) {
-        startHistoryScan(scanAddress, startHeight, endHeight, subscriptionStatusRef, true);
+    private void emitMetrics() {
+        try {
+            metrics.format(metrics.snapshotAndReset()).ifPresent(log::info);
+        } catch(Throwable t) {
+            log.warn("Failed to emit scan metrics", t);
+        }
     }
 
-    public void startHistoryScan(SilentPaymentScanAddress scanAddress, Integer startHeight, Integer endHeight, WeakReference<SubscriptionStatus> subscriptionStatusRef, boolean postIfEmpty) {
-        queryPool.submit(() -> {
-            SilentPaymentsSubscription subscription = new SilentPaymentsSubscription(scanAddress.toString(), startHeight == null ? 0 : startHeight);
-            List<TxEntry> history = blocksIndex.getHistoryAsync(scanAddress, subscription, startHeight, endHeight, subscriptionStatusRef);
-            List<TxEntry> mempoolHistory = getMempoolHistory(scanAddress, subscriptionStatusRef, subscription);
-            history.addAll(mempoolHistory);
+    public void close() {
+        if(metricsExecutor != null) {
+            metricsExecutor.shutdownNow();
+        }
+    }
 
-            if(postIfEmpty || !history.isEmpty()) {
-                Frigate.getEventBus().post(new SilentPaymentsNotification(subscription, PROGRESS_COMPLETE, new ArrayList<>(history), subscriptionStatusRef.get()));
+    public void startHistoryScan(SilentPaymentScanAddress scanAddress, Integer startHeight, Integer endHeight, SilentPaymentAddressSubscription subscription, WeakReference<SubscriptionStatus> subscriptionStatusRef, boolean isHistorical) {
+        BooleanSupplier cancelled = subscription.captureScanCancellation();
+        queryPool.submit(() -> {
+            long startMillis = isHistorical ? System.currentTimeMillis() : 0L;
+            try {
+                SilentPaymentsSubscription notificationSubscription = new SilentPaymentsSubscription(scanAddress.toString(), subscription.getLabels().toArray(new Integer[0]), subscription.getStartHeight());
+                List<SilentPaymentsTxEntry> history = blocksIndex.getHistoryAsync(scanAddress, notificationSubscription, startHeight, endHeight, null, subscriptionStatusRef, cancelled, isHistorical);
+                List<SilentPaymentsTxEntry> mempoolHistory = getMempoolHistory(scanAddress, null, subscriptionStatusRef, notificationSubscription, cancelled);
+                history.addAll(mempoolHistory);
+                long scanDurationMillis = isHistorical ? System.currentTimeMillis() - startMillis : 0L;
+
+                boolean wasCancelled = cancelled.getAsBoolean();
+                if(!wasCancelled && (isHistorical || !history.isEmpty())) {
+                    Frigate.getEventBus().post(new SilentPaymentsNotification(notificationSubscription, PROGRESS_COMPLETE, new ArrayList<>(history), subscriptionStatusRef.get()));
+                }
+                if(!wasCancelled && isHistorical) {
+                    subscription.markHistoricalComplete();
+                    if(metrics != null) {
+                        metrics.record(history.size(), scanDurationMillis);
+                    }
+                }
+            } catch(Throwable t) {
+                log.error("History scan task failed for " + scanAddress + " (start=" + startHeight + ", end=" + endHeight + ", isHistorical=" + isHistorical + ")", t);
             }
         });
     }
 
-    public void startMempoolScan(SilentPaymentScanAddress scanAddress, Integer startHeight, Integer endHeight, WeakReference<SubscriptionStatus> subscriptionStatusRef) {
+    public void startMempoolScan(SilentPaymentScanAddress scanAddress, Integer startHeight, Integer endHeight, Set<Sha256Hash> mempoolTxids, SilentPaymentAddressSubscription subscription, WeakReference<SubscriptionStatus> subscriptionStatusRef) {
+        BooleanSupplier cancelled = subscription.captureScanCancellation();
         queryPool.submit(() -> {
-            SilentPaymentsSubscription subscription = new SilentPaymentsSubscription(scanAddress.toString(), startHeight == null ? 0 : startHeight);
-            List<TxEntry> mempoolHistory = getMempoolHistory(scanAddress, subscriptionStatusRef, subscription);
+            try {
+                SilentPaymentsSubscription notificationSubscription = new SilentPaymentsSubscription(scanAddress.toString(), subscription.getLabels().toArray(new Integer[0]), subscription.getStartHeight());
+                List<SilentPaymentsTxEntry> mempoolHistory = getMempoolHistory(scanAddress, mempoolTxids, subscriptionStatusRef, notificationSubscription, cancelled);
 
-            if(!mempoolHistory.isEmpty()) {
-                Frigate.getEventBus().post(new SilentPaymentsNotification(subscription, PROGRESS_COMPLETE, new ArrayList<>(mempoolHistory), subscriptionStatusRef.get()));
+                if(!cancelled.getAsBoolean() && !mempoolHistory.isEmpty()) {
+                    Frigate.getEventBus().post(new SilentPaymentsNotification(notificationSubscription, PROGRESS_COMPLETE, new ArrayList<>(mempoolHistory), subscriptionStatusRef.get()));
+                }
+            } catch(Throwable t) {
+                log.error("Mempool scan task failed for " + scanAddress, t);
             }
         });
     }
 
-    private List<TxEntry> getMempoolHistory(SilentPaymentScanAddress scanAddress, WeakReference<SubscriptionStatus> subscriptionStatusRef, SilentPaymentsSubscription subscription) {
-        List<TxEntry> mempoolHistory = mempoolIndex.getHistoryAsync(scanAddress, subscription, null, null, subscriptionStatusRef);
+    private List<SilentPaymentsTxEntry> getMempoolHistory(SilentPaymentScanAddress scanAddress, Set<Sha256Hash> mempoolTxids, WeakReference<SubscriptionStatus> subscriptionStatusRef, SilentPaymentsSubscription notificationSubscription, BooleanSupplier cancelled) {
+        List<SilentPaymentsTxEntry> mempoolHistory = mempoolIndex.getHistoryAsync(scanAddress, notificationSubscription, null, null, mempoolTxids, subscriptionStatusRef, cancelled, false);
         SubscriptionStatus subscriptionStatus = subscriptionStatusRef.get();
         if(subscriptionStatus != null && subscriptionStatus.getSilentPaymentsMempoolTxids(scanAddress.toString()) != null) {
             mempoolHistory.removeIf(txEntry -> subscriptionStatus.getSilentPaymentsMempoolTxids(scanAddress.toString()).contains(Sha256Hash.wrap(txEntry.tx_hash)));

src/main/java/com/sparrowwallet/frigate/index/MemoryDbManager.java

@@ -13,13 +13,13 @@ import java.sql.Statement;
 import java.util.Properties;
 
 public class MemoryDbManager implements DbManager {
-    private final static Logger log = LoggerFactory.getLogger(ScalingDbManager.class);
+    private final static Logger log = LoggerFactory.getLogger(MemoryDbManager.class);
 
     private Connection connection;
     private boolean shutdown = false;
 
     @Override
-    public <T> T executeRead(ReadOperation<T> operation) throws SQLException {
+    public synchronized <T> T executeRead(ReadOperation<T> operation) throws SQLException {
         if(shutdown) {
             throw new SQLException("Connection manager is shutting down");
         }
@@ -29,7 +29,7 @@ public class MemoryDbManager implements DbManager {
     }
 
     @Override
-    public <T> T executeWrite(WriteOperation<T> operation) throws SQLException {
+    public synchronized <T> T executeWrite(WriteOperation<T> operation) throws SQLException {
         if(shutdown) {
             throw new SQLException("Connection manager is shutting down");
         }
@@ -63,15 +63,18 @@ public class MemoryDbManager implements DbManager {
 
         Properties duckDbProperties = new Properties();
         duckDbProperties.setProperty("allow_unsigned_extensions", "true");
-        if(Config.get().getDbThreads() != null) {
-            duckDbProperties.setProperty("threads", Config.get().getDbThreads().toString());
+        if(Config.get().getScan().getDbThreads() != null) {
+            duckDbProperties.setProperty("threads", Config.get().getScan().getDbThreads().toString());
+        }
+        if(Config.get().getScan().getMemoryLimit() != null) {
+            duckDbProperties.setProperty("memory_limit", Config.get().getScan().getMemoryLimit());
         }
 
         connection = DriverManager.getConnection(DbManager.DB_PREFIX + "memory:", duckDbProperties);
 
-        File secp256k1ExtensionFile = Storage.getSecp256k1ExtensionFile();
+        File ufsecpExtensionFile = Storage.getUfsecpExtensionFile();
         try(Statement statement = connection.createStatement()) {
-            statement.execute("LOAD '" + secp256k1ExtensionFile.getAbsolutePath() + "';");
+            statement.execute("LOAD '" + ufsecpExtensionFile.getAbsolutePath() + "';");
         }
     }
 }

src/main/java/com/sparrowwallet/frigate/index/ScalingDbManager.java

@@ -1,6 +1,5 @@
 package com.sparrowwallet.frigate.index;
 
-import com.zaxxer.hikari.HikariDataSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -15,15 +14,19 @@ public class ScalingDbManager extends AbstractDbManager {
 
     private final String readWriteUrl;
     private Connection writeConnection;
-    private final List<HikariDataSource> dataSources = new ArrayList<>();
+    private final List<DuckDBReadPool> readPools = new ArrayList<>();
     private final AtomicInteger index = new AtomicInteger(0);
     private boolean shutdown = false;
 
     public ScalingDbManager(String readWriteUrl, List<String> readOnlyUrls) {
+        super();
         this.readWriteUrl = readWriteUrl;
         for(String url : readOnlyUrls) {
-            HikariDataSource ds = createReadDataSource(url, 1);
-            dataSources.add(ds);
+            try {
+                readPools.add(new DuckDBReadPool(url, 10));
+            } catch(SQLException e) {
+                throw new RuntimeException("Failed to create DuckDB read pool for " + url, e);
+            }
         }
     }
 
@@ -33,9 +36,17 @@ public class ScalingDbManager extends AbstractDbManager {
             throw new SQLException("Connection manager is shutting down");
         }
 
-        int ind = index.getAndIncrement() % dataSources.size();
-        HikariDataSource ds = dataSources.get(ind);
-        return operation.execute(ds.getConnection());
+        int ind = index.getAndIncrement() % readPools.size();
+        DuckDBReadPool pool = readPools.get(ind);
+        Connection conn = null;
+        try {
+            conn = pool.getConnection();
+            return operation.execute(conn);
+        } finally {
+            if(conn != null) {
+                pool.releaseConnection(conn);
+            }
+        }
     }
 
     @Override
@@ -60,8 +71,8 @@ public class ScalingDbManager extends AbstractDbManager {
             log.error("Error closing write connection", e);
         }
 
-        for(HikariDataSource ds : dataSources) {
-            ds.close();
+        for(DuckDBReadPool pool : readPools) {
+            pool.close();
         }
     }
 

src/main/java/com/sparrowwallet/frigate/index/SingleDbManager.java

@@ -1,6 +1,5 @@
 package com.sparrowwallet.frigate.index;
 
-import com.zaxxer.hikari.HikariDataSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -19,18 +18,20 @@ public class SingleDbManager extends AbstractDbManager {
     private final Semaphore writerWaiting;
 
     private Connection writeConnection;
-    private HikariDataSource readDataSource;
+    private DuckDBReadPool readPool;
     private boolean inWriteMode;
     private volatile boolean shutdown = false;
     private volatile boolean writeOperationActive = false;
 
     public SingleDbManager(String connectionUrl) {
+        super();
         this.connectionUrl = connectionUrl;
         this.rwLock = new ReentrantReadWriteLock(true);
         this.writerWaiting = new Semaphore(1);
         this.inWriteMode = false;
     }
 
+    //Operations passed to executeRead must not await an operation that itself calls executeRead on another thread
     public <T> T executeRead(DbManager.ReadOperation<T> operation) throws SQLException, InterruptedException {
         if(shutdown) {
             throw new SQLException("Connection manager is shutting down");
@@ -49,12 +50,15 @@ public class SingleDbManager extends AbstractDbManager {
         }
 
         rwLock.readLock().lock();
+        Connection conn = null;
         try {
             ensureReadMode();
-            try(Connection conn = readDataSource.getConnection()) {
-                return operation.execute(conn);
-            }
+            conn = readPool.getConnection();
+            return operation.execute(conn);
         } finally {
+            if(conn != null) {
+                readPool.releaseConnection(conn);
+            }
             rwLock.readLock().unlock();
         }
     }
@@ -92,11 +96,11 @@ public class SingleDbManager extends AbstractDbManager {
     }
 
     private synchronized void ensureReadMode() throws SQLException {
-        if(inWriteMode || readDataSource == null) {
+        if(inWriteMode || readPool == null) {
             log.debug("Switching to READ mode");
             waitForWriteOperationToComplete();
             closeWriteConnection();
-            createReadDataSource();
+            createReadPool();
             inWriteMode = false;
         }
     }
@@ -104,7 +108,7 @@ public class SingleDbManager extends AbstractDbManager {
     private synchronized void ensureWriteMode() throws SQLException {
         if(!inWriteMode || writeConnection == null) {
             log.debug("Switching to WRITE mode");
-            closeReadDataSource();
+            closeReadPool();
             createWriteConnection();
             inWriteMode = true;
         }
@@ -124,12 +128,16 @@ public class SingleDbManager extends AbstractDbManager {
         }
     }
 
-    private void createReadDataSource() {
-        if(readDataSource != null) {
+    private void createReadPool() {
+        if(readPool != null) {
             return;
         }
 
-        readDataSource = createReadDataSource(connectionUrl, 10);
+        try {
+            readPool = new DuckDBReadPool(connectionUrl, 10);
+        } catch(SQLException e) {
+            throw new RuntimeException("Failed to create DuckDB read pool", e);
+        }
     }
 
     private void createWriteConnection() throws SQLException {
@@ -155,12 +163,12 @@ public class SingleDbManager extends AbstractDbManager {
         }
     }
 
-    private void closeReadDataSource() {
-        if(readDataSource != null) {
+    private void closeReadPool() {
+        if(readPool != null) {
             try {
-                readDataSource.close();
+                readPool.close();
             } finally {
-                readDataSource = null;
+                readPool = null;
                 log.debug("Closed read connection pool");
             }
         }
@@ -209,24 +217,16 @@ public class SingleDbManager extends AbstractDbManager {
         }
 
         try {
-            closeReadDataSource();
+            closeReadPool();
         } catch(Exception e) {
-            log.error("Error closing read data source", e);
+            log.error("Error closing read pool", e);
         }
 
         log.debug("Shutdown complete");
     }
 
-    public int getActiveReadConnections() {
-        return readDataSource != null ? readDataSource.getHikariPoolMXBean().getActiveConnections() : 0;
-    }
-
-    public int getIdleReadConnections() {
-        return readDataSource != null ? readDataSource.getHikariPoolMXBean().getIdleConnections() : 0;
-    }
-
-    public int getTotalReadConnections() {
-        return readDataSource != null ? readDataSource.getHikariPoolMXBean().getTotalConnections() : 0;
+    public boolean isReadPoolActive() {
+        return readPool != null;
     }
 
     public boolean isShutdown() {

src/main/java/com/sparrowwallet/frigate/io/ComputeBackend.java

@@ -0,0 +1,9 @@
+package com.sparrowwallet.frigate.io;
+
+public enum ComputeBackend {
+    AUTO, GPU, CPU;
+
+    public String toSqlValue() {
+        return name().toLowerCase();
+    }
+}

src/main/java/com/sparrowwallet/frigate/io/Config.java

@@ -1,206 +1,689 @@
 package com.sparrowwallet.frigate.io;
 
-import com.google.gson.*;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonSetter;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.toml.TomlMapper;
+import com.sparrowwallet.frigate.ConfigurationException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
-import java.lang.reflect.Type;
+import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.List;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
 public class Config {
     private static final Logger log = LoggerFactory.getLogger(Config.class);
 
-    public static final String CONFIG_FILENAME = "config";
+    public static final String TOML_CONFIG_FILENAME = "config.toml";
+    public static final String JSON_CONFIG_FILENAME = "config";
 
-    private Server coreServer;
-    private CoreAuthType coreAuthType;
-    private File coreDataDir;
-    private String coreAuth;
-    private Boolean startIndexing;
-    private Integer indexStartHeight;
-    private Integer scriptPubKeyCacheSize;
-    private Integer dbThreads;
-    private String dbUrl;
-    private List<String> readDbUrls;
+    private CoreConfig core;
+    private IndexConfig index;
+    private ScanConfig scan;
+    private ServerConfig server;
+    private DatabaseConfig database;
 
     private static Config INSTANCE;
 
-    private static Gson getGson() {
-        GsonBuilder gsonBuilder = new GsonBuilder();
-        gsonBuilder.registerTypeAdapter(File.class, new FileSerializer());
-        gsonBuilder.registerTypeAdapter(File.class, new FileDeserializer());
-        gsonBuilder.registerTypeAdapter(Server.class, new ServerSerializer());
-        gsonBuilder.registerTypeAdapter(Server.class, new ServerDeserializer());
-        return gsonBuilder.setPrettyPrinting().disableHtmlEscaping().create();
+    private static final TomlMapper tomlMapper = TomlMapper.builder().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).build();
+
+    public Config() {
+        core = new CoreConfig();
+        index = new IndexConfig();
+        scan = new ScanConfig();
+        server = new ServerConfig();
+        database = new DatabaseConfig();
     }
 
-    private static File getConfigFile() {
-        File sparrowDir = Storage.getFrigateDir();
-        return new File(sparrowDir, CONFIG_FILENAME);
+    private static File getTomlConfigFile() {
+        return new File(Storage.getFrigateDir(), TOML_CONFIG_FILENAME);
+    }
+
+    private static File getJsonConfigFile() {
+        return new File(Storage.getFrigateDir(), JSON_CONFIG_FILENAME);
     }
 
     private static Config load() {
-        File configFile = getConfigFile();
-        if(configFile.exists()) {
-            try {
-                Reader reader = new FileReader(configFile);
-                Config config = getGson().fromJson(reader, Config.class);
-                reader.close();
+        File tomlFile = getTomlConfigFile();
+        File jsonFile = getJsonConfigFile();
 
+        if(tomlFile.exists()) {
+            try {
+                Config config = tomlMapper.readValue(tomlFile, Config.class);
                 if(config != null) {
                     return config;
                 }
             } catch(Exception e) {
-                log.error("Error opening " + configFile.getAbsolutePath(), e);
-                //Ignore and assume no config
+                log.error("Error reading " + tomlFile.getAbsolutePath(), e);
+            }
+        } else if(jsonFile.exists()) {
+            try {
+                Config config = migrateFromJson(jsonFile);
+                if(config != null) {
+                    saveToml(config, tomlFile);
+                    File backupFile = new File(jsonFile.getPath() + ".bak");
+                    jsonFile.renameTo(backupFile);
+                    log.info("Migrated config from JSON to TOML (backup: {})", backupFile.getName());
+                    return config;
+                }
+            } catch(Exception e) {
+                log.error("Error migrating " + jsonFile.getAbsolutePath(), e);
+            }
+        } else {
+            try {
+                writeDefaultConfig(tomlFile);
+            } catch(Exception e) {
+                log.error("Error writing default config", e);
             }
         }
 
         return new Config();
     }
 
+    private static Config migrateFromJson(File jsonFile) throws IOException {
+        ObjectMapper jsonMapper = new ObjectMapper();
+        jsonMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        JsonNode root = jsonMapper.readTree(jsonFile);
+
+        Config config = new Config();
+
+        if(root.has("coreServer")) {
+            config.getCore().setServer(root.get("coreServer").asText());
+        }
+        if(root.has("coreAuthType")) {
+            config.getCore().setAuthType(root.get("coreAuthType").asText());
+        }
+        if(root.has("coreDataDir")) {
+            config.getCore().setDataDir(root.get("coreDataDir").asText());
+        }
+        if(root.has("coreAuth")) {
+            config.getCore().setAuth(root.get("coreAuth").asText());
+        }
+        if(root.has("startIndexing")) {
+            config.getCore().setConnect(root.get("startIndexing").asBoolean());
+        }
+        if(root.has("indexStartHeight")) {
+            config.getIndex().setStartHeight(root.get("indexStartHeight").asInt());
+        }
+        if(root.has("scriptPubKeyCacheSize")) {
+            int oldSize = root.get("scriptPubKeyCacheSize").asInt();
+            config.getIndex().setCacheSize(formatCacheSize(oldSize));
+        }
+
+        if(root.has("batchSize")) {
+            config.getScan().setBatchSize(root.get("batchSize").asInt());
+        }
+        if(root.has("computeBackend")) {
+            config.getScan().setComputeBackend(root.get("computeBackend").asText());
+        }
+        if(root.has("dbThreads")) {
+            config.getScan().setDbThreads(root.get("dbThreads").asInt());
+        }
+
+        if(root.has("backendElectrumServer")) {
+            config.getServer().setBackendElectrumServer(root.get("backendElectrumServer").asText());
+        }
+
+        if(root.has("dbUrl")) {
+            config.getDatabase().setUrl(root.get("dbUrl").asText());
+        }
+        if(root.has("readDbUrls")) {
+            List<String> urls = new java.util.ArrayList<>();
+            root.get("readDbUrls").forEach(node -> urls.add(node.asText()));
+            config.getDatabase().setReadUrls(urls);
+        }
+
+        return config;
+    }
+
+    private static String formatCacheSize(int size) {
+        if(size >= 1_000_000 && size % 1_000_000 == 0) {
+            return (size / 1_000_000) + "M";
+        } else if(size >= 1_000 && size % 1_000 == 0) {
+            return (size / 1_000) + "K";
+        }
+        return String.valueOf(size);
+    }
+
+    static int parseCacheSize(String value) {
+        if(value == null || value.isEmpty()) {
+            return 10_000_000;
+        }
+        String s = value.trim().toUpperCase();
+        if(s.endsWith("M")) {
+            return (int) (Double.parseDouble(s.substring(0, s.length() - 1)) * 1_000_000);
+        } else if(s.endsWith("K")) {
+            return (int) (Double.parseDouble(s.substring(0, s.length() - 1)) * 1_000);
+        }
+        return Integer.parseInt(s);
+    }
+
+    private static void writeDefaultConfig(File tomlFile) {
+        try(InputStream is = Config.class.getResourceAsStream("/config.toml.default")) {
+            if(is != null) {
+                Storage.createOwnerOnlyFile(tomlFile);
+                Files.copy(is, tomlFile.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING);
+            }
+        } catch(IOException e) {
+            log.error("Error writing default config", e);
+        }
+    }
+
+    private static void saveToml(Config config, File tomlFile) {
+        try {
+            if(!tomlFile.exists()) {
+                Storage.createOwnerOnlyFile(tomlFile);
+            }
+            tomlMapper.writeValue(tomlFile, config);
+        } catch(IOException e) {
+            log.error("Error writing config", e);
+        }
+    }
+
     public static synchronized Config get() {
         if(INSTANCE == null) {
             INSTANCE = load();
+            INSTANCE.getServer().getAdvertisedHosts();
         }
-
         return INSTANCE;
     }
 
-    public Server getCoreServer() {
-        return coreServer;
+    public CoreConfig getCore() {
+        if(core == null) {
+            core = new CoreConfig();
+        }
+        return core;
     }
 
-    public void setCoreServer(Server coreServer) {
-        this.coreServer = coreServer;
-        flush();
+    public IndexConfig getIndex() {
+        if(index == null) {
+            index = new IndexConfig();
+        }
+        return index;
     }
 
-    public CoreAuthType getCoreAuthType() {
-        return coreAuthType;
+    public ScanConfig getScan() {
+        if(scan == null) {
+            scan = new ScanConfig();
+        }
+        return scan;
     }
 
-    public void setCoreAuthType(CoreAuthType coreAuthType) {
-        this.coreAuthType = coreAuthType;
-        flush();
+    public ServerConfig getServer() {
+        if(server == null) {
+            server = new ServerConfig();
+        }
+        return server;
     }
 
-    public File getCoreDataDir() {
-        return coreDataDir;
+    public DatabaseConfig getDatabase() {
+        if(database == null) {
+            database = new DatabaseConfig();
+        }
+        return database;
     }
 
-    public void setCoreDataDir(File coreDataDir) {
-        this.coreDataDir = coreDataDir;
-        flush();
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public static class CoreConfig {
+        public static final int DEFAULT_RPC_REQUEST_TIMEOUT_SECONDS = 60;
+        public static final int DEFAULT_RPC_BATCH_SIZE = 100;
+
+        private Boolean connect;
+        private String server;
+        private String authType;
+        private String dataDir;
+        private String auth;
+        private String zmqSequenceEndpoint;
+        private Integer rpcRequestTimeoutSeconds;
+        private Integer rpcBatchSize;
+
+        public Boolean getConnect() {
+            return connect;
+        }
+
+        public void setConnect(Boolean connect) {
+            this.connect = connect;
+        }
+
+        @JsonIgnore
+        public boolean shouldConnect() {
+            return connect == null || connect;
+        }
+
+        public String getServer() {
+            return server;
+        }
+
+        public void setServer(String server) {
+            this.server = server;
+        }
+
+        public String getAuthType() {
+            return authType;
+        }
+
+        public void setAuthType(String authType) {
+            this.authType = authType;
+        }
+
+        @JsonIgnore
+        public CoreAuthType getAuthTypeEnum() {
+            if(authType == null) {
+                return null;
+            }
+            try {
+                return CoreAuthType.valueOf(authType);
+            } catch(Exception e) {
+                return null;
+            }
+        }
+
+        public String getDataDir() {
+            return dataDir;
+        }
+
+        public void setDataDir(String dataDir) {
+            this.dataDir = dataDir;
+        }
+
+        @JsonIgnore
+        public File getDataDirFile() {
+            return dataDir != null ? new File(dataDir) : null;
+        }
+
+        public String getAuth() {
+            return auth;
+        }
+
+        public void setAuth(String auth) {
+            this.auth = auth;
+        }
+
+        @JsonIgnore
+        public Server getServerObj() {
+            return server != null ? Server.fromString(server) : null;
+        }
+
+        public String getZmqSequenceEndpoint() {
+            return zmqSequenceEndpoint;
+        }
+
+        public void setZmqSequenceEndpoint(String zmqSequenceEndpoint) {
+            this.zmqSequenceEndpoint = zmqSequenceEndpoint;
+        }
+
+        public Integer getRpcRequestTimeoutSeconds() {
+            return rpcRequestTimeoutSeconds;
+        }
+
+        public void setRpcRequestTimeoutSeconds(Integer rpcRequestTimeoutSeconds) {
+            this.rpcRequestTimeoutSeconds = rpcRequestTimeoutSeconds;
+        }
+
+        @JsonIgnore
+        public int getRpcRequestTimeoutMillis() {
+            int seconds = rpcRequestTimeoutSeconds != null ? rpcRequestTimeoutSeconds : DEFAULT_RPC_REQUEST_TIMEOUT_SECONDS;
+            return seconds * 1000;
+        }
+
+        public Integer getRpcBatchSize() {
+            return rpcBatchSize;
+        }
+
+        public void setRpcBatchSize(Integer rpcBatchSize) {
+            this.rpcBatchSize = rpcBatchSize;
+        }
+
+        @JsonIgnore
+        public int getRpcBatchSizeValue() {
+            return rpcBatchSize != null && rpcBatchSize > 0 ? rpcBatchSize : DEFAULT_RPC_BATCH_SIZE;
+        }
     }
 
-    public String getCoreAuth() {
-        return coreAuth;
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public static class IndexConfig {
+        private Integer startHeight;
+        private String cacheSize;
+
+        public Integer getStartHeight() {
+            return startHeight;
+        }
+
+        public void setStartHeight(Integer startHeight) {
+            this.startHeight = startHeight;
+        }
+
+        public String getCacheSize() {
+            return cacheSize;
+        }
+
+        public void setCacheSize(String cacheSize) {
+            this.cacheSize = cacheSize;
+        }
+
+        @JsonIgnore
+        public int getCacheSizeEntries() {
+            return Config.parseCacheSize(cacheSize);
+        }
     }
 
-    public void setCoreAuth(String coreAuth) {
-        this.coreAuth = coreAuth;
-        flush();
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public static class ScanConfig {
+        public static final int DEFAULT_BATCH_SIZE = 300_000;
+        public static final int DEFAULT_MAX_LABELS = 10;
+        public static final int DEFAULT_MAX_SUBSCRIPTIONS = 100;
+
+        private Integer batchSize;
+        private String computeBackend;
+        private Integer dbThreads;
+        private String memoryLimit;
+        private Integer maxLabels;
+        private Integer maxSubscriptions;
+        private Boolean metricsEnabled;
+
+        public int getBatchSize() {
+            return batchSize != null ? batchSize : DEFAULT_BATCH_SIZE;
+        }
+
+        public void setBatchSize(int batchSize) {
+            this.batchSize = batchSize;
+        }
+
+        public int getMaxLabels() {
+            return maxLabels != null ? maxLabels : DEFAULT_MAX_LABELS;
+        }
+
+        public void setMaxLabels(Integer maxLabels) {
+            this.maxLabels = maxLabels;
+        }
+
+        public int getMaxSubscriptions() {
+            return maxSubscriptions != null ? maxSubscriptions : DEFAULT_MAX_SUBSCRIPTIONS;
+        }
+
+        public void setMaxSubscriptions(Integer maxSubscriptions) {
+            this.maxSubscriptions = maxSubscriptions;
+        }
+
+        @JsonIgnore
+        public ComputeBackend getComputeBackendEnum() {
+            if(computeBackend == null) {
+                return ComputeBackend.AUTO;
+            }
+            try {
+                return ComputeBackend.valueOf(computeBackend);
+            } catch(Exception e) {
+                return ComputeBackend.AUTO;
+            }
+        }
+
+        public String getComputeBackend() {
+            return computeBackend;
+        }
+
+        public void setComputeBackend(String computeBackend) {
+            this.computeBackend = computeBackend;
+        }
+
+        public Integer getDbThreads() {
+            return dbThreads;
+        }
+
+        public void setDbThreads(Integer dbThreads) {
+            this.dbThreads = dbThreads;
+        }
+
+        public String getMemoryLimit() {
+            return memoryLimit;
+        }
+
+        public void setMemoryLimit(String memoryLimit) {
+            this.memoryLimit = memoryLimit;
+        }
+
+        public boolean isMetricsEnabled() {
+            return metricsEnabled == null || metricsEnabled;
+        }
+
+        public void setMetricsEnabled(Boolean metricsEnabled) {
+            this.metricsEnabled = metricsEnabled;
+        }
     }
 
-    public Boolean isStartIndexing() {
-        return startIndexing;
-    }
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public static class ServerConfig {
+        private List<String> host;
+        private String tcp;
+        private String ssl;
+        private String sslCert;
+        private String sslKey;
+        private String backendElectrumServer;
 
-    public void setStartIndexing(Boolean startIndexing) {
-        this.startIndexing = startIndexing;
-        flush();
-    }
+        @JsonIgnore
+        private List<Server> advertisedHostsCache;
 
-    public Integer getIndexStartHeight() {
-        return indexStartHeight;
-    }
+        public Object getHost() {
+            if(host == null || host.isEmpty()) {
+                return null;
+            }
+            return host.size() == 1 ? host.getFirst() : host;
+        }
 
-    public void setIndexStartHeight(Integer indexStartHeight) {
-        this.indexStartHeight = indexStartHeight;
-        flush();
-    }
-
-    public Integer getScriptPubKeyCacheSize() {
-        return scriptPubKeyCacheSize;
-    }
-
-    public void setScriptPubKeyCacheSize(Integer scriptPubKeyCacheSize) {
-        this.scriptPubKeyCacheSize = scriptPubKeyCacheSize;
-        flush();
-    }
-
-    public Integer getDbThreads() {
-        return dbThreads;
-    }
-
-    public void setDbThreads(Integer dbThreads) {
-        this.dbThreads = dbThreads;
-        flush();
-    }
-
-    public String getDbUrl() {
-        return dbUrl;
-    }
-
-    public void setDbUrl(String dbUrl) {
-        this.dbUrl = dbUrl;
-        flush();
-    }
-
-    public List<String> getReadDbUrls() {
-        return readDbUrls;
-    }
-
-    public void setReadDbUrls(List<String> readDbUrls) {
-        this.readDbUrls = readDbUrls;
-        flush();
-    }
-
-    private synchronized void flush() {
-        Gson gson = getGson();
-        try {
-            File configFile = getConfigFile();
-            if(!configFile.exists()) {
-                Storage.createOwnerOnlyFile(configFile);
+        @JsonSetter("host")
+        public void setHost(JsonNode node) {
+            if(node == null || node.isNull()) {
+                this.host = null;
+                return;
             }
 
-            Writer writer = new FileWriter(configFile);
-            gson.toJson(this, writer);
-            writer.flush();
-            writer.close();
-        } catch (IOException e) {
-            //Ignore
+            List<String> entries = new ArrayList<>();
+            if(node.isTextual()) {
+                String value = node.asText();
+                if(!value.isEmpty()) {
+                    entries.add(value);
+                }
+            } else if(node.isArray()) {
+                for(JsonNode item : node) {
+                    if(!item.isTextual()) {
+                        throw new ConfigurationException("Each entry in 'host' under [server] must be a string (got: " + item + ")");
+                    }
+                    String value = item.asText();
+                    if(!value.isEmpty()) {
+                        entries.add(value);
+                    }
+                }
+            } else {
+                throw new ConfigurationException("'host' under [server] must be a string or array of strings (got: " + node + ")");
+            }
+
+            this.host = entries.isEmpty() ? null : entries;
+        }
+
+        @JsonIgnore
+        public List<Server> getAdvertisedHosts() {
+            if(advertisedHostsCache != null) {
+                return advertisedHostsCache;
+            }
+            if(host == null || host.isEmpty()) {
+                advertisedHostsCache = List.of();
+                return advertisedHostsCache;
+            }
+
+            List<Server> result = new ArrayList<>();
+            for(String entry : host) {
+                Protocol protocol = Protocol.getProtocol(entry);
+                if(protocol == null) {
+                    Server tcpBind = getTcpServer();
+                    if(tcpBind != null) {
+                        result.add(Server.fromString(Protocol.TCP.toUrlString(entry, tcpBind.getHostAndPort().getPort())));
+                    }
+                    Server sslBind = getSslServer();
+                    if(sslBind != null) {
+                        result.add(Server.fromString(Protocol.SSL.toUrlString(entry, sslBind.getHostAndPort().getPort())));
+                    }
+                } else if(protocol == Protocol.TCP || protocol == Protocol.SSL) {
+                    result.add(parseListener("host", entry, protocol));
+                } else {
+                    throw new ConfigurationException("Host advertisement '" + entry + "' under [server] must use tcp:// or ssl:// scheme");
+                }
+            }
+
+            advertisedHostsCache = List.copyOf(result);
+            return advertisedHostsCache;
+        }
+
+        public String getTcp() {
+            return tcp;
+        }
+
+        public void setTcp(String tcp) {
+            this.tcp = tcp;
+        }
+
+        @JsonSetter("port")
+        public void setLegacyPort(Integer port) {
+            if(port != null && this.tcp == null) {
+                this.tcp = "tcp://0.0.0.0:" + port;
+            }
+        }
+
+        public String getSsl() {
+            return ssl;
+        }
+
+        public void setSsl(String ssl) {
+            this.ssl = ssl;
+        }
+
+        @JsonIgnore
+        public Server getTcpServer() {
+            if(tcp == null && ssl == null) {
+                return parseListener("tcp", "tcp://0.0.0.0:" + Protocol.TCP.getDefaultPort(), Protocol.TCP);
+            }
+            if(tcp == null || tcp.isEmpty()) {
+                return null;
+            }
+            return parseListener("tcp", tcp, Protocol.TCP);
+        }
+
+        @JsonIgnore
+        public Server getSslServer() {
+            if(ssl == null || ssl.isEmpty()) {
+                return null;
+            }
+            return parseListener("ssl", ssl, Protocol.SSL);
+        }
+
+        @JsonIgnore
+        public boolean isTcpEnabled() {
+            return getTcpServer() != null;
+        }
+
+        @JsonIgnore
+        public boolean isSslEnabled() {
+            return getSslServer() != null;
+        }
+
+        private static Server parseListener(String key, String url, Protocol expected) {
+            Server server;
+            try {
+                server = Server.fromString(url);
+            } catch(IllegalArgumentException e) {
+                throw new com.sparrowwallet.frigate.ConfigurationException("Invalid " + key + " listener URL '" + url + "' under [server] in config.toml: " + e.getMessage(), e);
+            }
+
+            if(server.getProtocol() != expected) {
+                throw new com.sparrowwallet.frigate.ConfigurationException("Listener '" + key + "' must use the " + expected.toUrlString() + " scheme (got '" + url + "')");
+            }
+
+            if(!server.getHostAndPort().hasPort()) {
+                throw new com.sparrowwallet.frigate.ConfigurationException("Listener '" + key + "' must specify an explicit port: '" + url + "'");
+            }
+
+            return server;
+        }
+
+        public static final String DEFAULT_SSL_CERT = "cert.pem";
+        public static final String DEFAULT_SSL_KEY = "key.pem";
+
+        public String getSslCert() {
+            return sslCert;
+        }
+
+        public void setSslCert(String sslCert) {
+            this.sslCert = sslCert;
+        }
+
+        @JsonIgnore
+        public File getSslCertFile() {
+            return resolveFrigateDirPath(sslCert, DEFAULT_SSL_CERT);
+        }
+
+        public String getSslKey() {
+            return sslKey;
+        }
+
+        public void setSslKey(String sslKey) {
+            this.sslKey = sslKey;
+        }
+
+        @JsonIgnore
+        public File getSslKeyFile() {
+            return resolveFrigateDirPath(sslKey, DEFAULT_SSL_KEY);
+        }
+
+        private static File resolveFrigateDirPath(String configured, String defaultName) {
+            String value = (configured == null || configured.isEmpty()) ? defaultName : configured;
+            File file = new File(value);
+            return file.isAbsolute() ? file : new File(Storage.getFrigateDir(), value);
+        }
+
+        public String getBackendElectrumServer() {
+            return backendElectrumServer;
+        }
+
+        public void setBackendElectrumServer(String backendElectrumServer) {
+            this.backendElectrumServer = backendElectrumServer;
+        }
+
+        @JsonIgnore
+        public Server getBackendElectrumServerObj() {
+            return backendElectrumServer != null ? Server.fromString(backendElectrumServer) : null;
         }
     }
 
-    private static class FileSerializer implements JsonSerializer<File> {
-        @Override
-        public JsonElement serialize(File src, Type typeOfSrc, JsonSerializationContext context) {
-            return new JsonPrimitive(src.getAbsolutePath());
-        }
-    }
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public static class DatabaseConfig {
+        private String url;
+        private List<String> readUrls;
 
-    private static class FileDeserializer implements JsonDeserializer<File> {
-        @Override
-        public File deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
-            return new File(json.getAsJsonPrimitive().getAsString());
+        public String getUrl() {
+            return url;
         }
-    }
 
-    private static class ServerSerializer implements JsonSerializer<Server> {
-        @Override
-        public JsonElement serialize(Server src, Type typeOfSrc, JsonSerializationContext context) {
-            return new JsonPrimitive(src.toString());
+        public void setUrl(String url) {
+            this.url = url;
         }
-    }
 
-    private static class ServerDeserializer implements JsonDeserializer<Server> {
-        @Override
-        public Server deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
-            return Server.fromString(json.getAsJsonPrimitive().getAsString());
+        public List<String> getReadUrls() {
+            return readUrls;
+        }
+
+        public void setReadUrls(List<String> readUrls) {
+            this.readUrls = readUrls;
         }
     }
 }
-

src/main/java/com/sparrowwallet/frigate/io/JsonRpcErrorFilter.java

@@ -0,0 +1,28 @@
+package com.sparrowwallet.frigate.io;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.turbo.TurboFilter;
+import ch.qos.logback.core.spi.FilterReply;
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcError;
+import org.slf4j.Marker;
+
+public class JsonRpcErrorFilter extends TurboFilter {
+    private static final String JSON_RPC_SERVER_LOGGER = "com.github.arteam.simplejsonrpc.server.JsonRpcServer";
+
+    @Override
+    public FilterReply decide(Marker marker, Logger logger, Level level, String format, Object[] params, Throwable t) {
+        if(level == Level.ERROR && t != null && JSON_RPC_SERVER_LOGGER.equals(logger.getName()) && rootCauseHasJsonRpcError(t)) {
+            return FilterReply.DENY;
+        }
+        return FilterReply.NEUTRAL;
+    }
+
+    private static boolean rootCauseHasJsonRpcError(Throwable t) {
+        Throwable cause = t;
+        while(cause.getCause() != null && cause.getCause() != cause) {
+            cause = cause.getCause();
+        }
+        return cause.getClass().isAnnotationPresent(JsonRpcError.class);
+    }
+}

src/main/java/com/sparrowwallet/frigate/io/RecentBlocksMap.java

@@ -65,14 +65,14 @@ public class RecentBlocksMap {
         try {
             File recentBlocksFile = getRecentBlocksFile();
             if(recentBlocksFile.exists()) {
-                Reader reader = new FileReader(recentBlocksFile);
-                Type mapType = new TypeToken<Map<Integer, String>>() {}.getType();
-                Map<Integer, String> loadedData = getGson().fromJson(reader, mapType);
-                if(loadedData != null) {
-                    this.data.putAll(loadedData);
-                    enforceMaxSize();
+                try(Reader reader = new FileReader(recentBlocksFile)) {
+                    Type mapType = new TypeToken<Map<Integer, String>>() {}.getType();
+                    Map<Integer, String> loadedData = getGson().fromJson(reader, mapType);
+                    if(loadedData != null) {
+                        this.data.putAll(loadedData);
+                        enforceMaxSize();
+                    }
                 }
-                reader.close();
             }
         } catch(IOException e) {
             // Ignore and start with empty map
@@ -87,10 +87,10 @@ public class RecentBlocksMap {
                 Storage.createOwnerOnlyFile(recentBlocksFile);
             }
 
-            Writer writer = new FileWriter(recentBlocksFile);
-            gson.toJson(this.data, writer);
-            writer.flush();
-            writer.close();
+            try(Writer writer = new FileWriter(recentBlocksFile)) {
+                gson.toJson(this.data, writer);
+                writer.flush();
+            }
         } catch(IOException e) {
             //Ignore
         }

src/main/java/com/sparrowwallet/frigate/io/Server.java

@@ -2,6 +2,9 @@ package com.sparrowwallet.frigate.io;
 
 import com.google.common.net.HostAndPort;
 
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 
 public class Server {
@@ -53,6 +56,11 @@ public class Server {
         return getHostAndPort().getHost();
     }
 
+    public InetSocketAddress getInetSocketAddress() throws UnknownHostException {
+        HostAndPort hostAndPort = getHostAndPort();
+        return new InetSocketAddress(InetAddress.getByName(hostAndPort.getHost()), hostAndPort.getPort());
+    }
+
     public boolean isOnionAddress() {
         return Protocol.isOnionAddress(getHostAndPort());
     }

src/main/java/com/sparrowwallet/frigate/io/SslUtil.java

@@ -0,0 +1,156 @@
+package com.sparrowwallet.frigate.io;
+
+import com.sparrowwallet.frigate.ConfigurationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.security.KeyFactory;
+import java.security.KeyStore;
+import java.security.PrivateKey;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public final class SslUtil {
+    private static final Logger log = LoggerFactory.getLogger(SslUtil.class);
+
+    private static final Pattern PEM_BLOCK = Pattern.compile("-----BEGIN ([A-Z0-9 ]+?)-----\\s*([A-Za-z0-9+/=\\s]+?)-----END \\1-----", Pattern.DOTALL);
+    private static final List<String> KEY_FACTORY_ALGORITHMS = List.of("RSA", "EC", "DSA");
+
+    private SslUtil() {}
+
+    public static SSLSocketFactory getTrustAllSocketFactory() {
+        TrustManager[] trustAllCerts = new TrustManager[] {
+                new X509TrustManager() {
+                    public X509Certificate[] getAcceptedIssuers() {
+                        return new X509Certificate[0];
+                    }
+
+                    public void checkClientTrusted(X509Certificate[] certs, String authType) throws CertificateException {
+                    }
+
+                    public void checkServerTrusted(X509Certificate[] certs, String authType) throws CertificateException {
+                    }
+                }
+        };
+
+        try {
+            SSLContext sslContext = SSLContext.getInstance("TLS");
+            sslContext.init(null, trustAllCerts, null);
+            return sslContext.getSocketFactory();
+        } catch(Exception e) {
+            log.error("Error creating SSL socket factory", e);
+        }
+
+        return null;
+    }
+
+    public static SSLContext getServerSSLContext(File certFile, File keyFile) {
+        if(!certFile.isFile()) {
+            throw new ConfigurationException("SSL: certificate file not found: " + certFile.getAbsolutePath());
+        }
+        if(!keyFile.isFile()) {
+            throw new ConfigurationException("SSL: private key file not found: " + keyFile.getAbsolutePath());
+        }
+
+        X509Certificate[] chain = readCertificateChain(certFile);
+        PrivateKey privateKey = readPrivateKey(keyFile);
+
+        try {
+            KeyStore keyStore = KeyStore.getInstance("PKCS12");
+            keyStore.load(null, new char[0]);
+            keyStore.setKeyEntry("frigate", privateKey, new char[0], chain);
+
+            KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
+            kmf.init(keyStore, new char[0]);
+
+            SSLContext sslContext = SSLContext.getInstance("TLS");
+            sslContext.init(kmf.getKeyManagers(), null, null);
+            return sslContext;
+        } catch(Exception e) {
+            throw new ConfigurationException("SSL: failed to initialise TLS context: " + e.getMessage(), e);
+        }
+    }
+
+    private static X509Certificate[] readCertificateChain(File certFile) {
+        try(FileInputStream fis = new FileInputStream(certFile);
+            BufferedInputStream bis = new BufferedInputStream(fis)) {
+            CertificateFactory cf = CertificateFactory.getInstance("X.509");
+            Collection<? extends Certificate> certs = cf.generateCertificates(bis);
+
+            if(certs.isEmpty()) {
+                throw new ConfigurationException("SSL: no certificates found in " + certFile.getAbsolutePath());
+            }
+
+            X509Certificate[] chain = new X509Certificate[certs.size()];
+            int i = 0;
+            for(Certificate c : certs) {
+                chain[i++] = (X509Certificate)c;
+            }
+
+            return chain;
+        } catch(IOException | CertificateException e) {
+            throw new ConfigurationException("SSL: failed to parse certificate " + certFile.getAbsolutePath() + ": " + e.getMessage(), e);
+        }
+    }
+
+    private static PrivateKey readPrivateKey(File keyFile) {
+        String pem;
+        try {
+            pem = Files.readString(keyFile.toPath(), StandardCharsets.UTF_8);
+        } catch(IOException e) {
+            throw new ConfigurationException("SSL: failed to read private key " + keyFile.getAbsolutePath() + ": " + e.getMessage(), e);
+        }
+
+        Matcher m = PEM_BLOCK.matcher(pem);
+        if(!m.find()) {
+            throw new ConfigurationException("SSL: no PEM block found in " + keyFile.getAbsolutePath());
+        }
+        String label = m.group(1).trim();
+        if(!"PRIVATE KEY".equals(label)) {
+            throw new ConfigurationException("SSL: unsupported key format '" + label + "' in " + keyFile.getAbsolutePath()
+                    + ". Only unencrypted PKCS#8 ('-----BEGIN PRIVATE KEY-----') is supported. Convert with: "
+                    + "openssl pkcs8 -topk8 -nocrypt -in <key> -out <pkcs8-key>");
+        }
+
+        byte[] der;
+        try {
+            der = Base64.getMimeDecoder().decode(m.group(2));
+        } catch(IllegalArgumentException e) {
+            throw new ConfigurationException("SSL: malformed base64 in private key " + keyFile.getAbsolutePath(), e);
+        }
+
+        PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(der);
+        InvalidKeySpecException lastException = null;
+        for(String algorithm : KEY_FACTORY_ALGORITHMS) {
+            try {
+                return KeyFactory.getInstance(algorithm).generatePrivate(spec);
+            } catch(InvalidKeySpecException e) {
+                lastException = e;
+            } catch(Exception e) {
+                throw new ConfigurationException("SSL: failed to load private key " + keyFile.getAbsolutePath() + ": " + e.getMessage(), e);
+            }
+        }
+
+        throw new ConfigurationException("SSL: unrecognised private key algorithm in " + keyFile.getAbsolutePath() + " (tried " + KEY_FACTORY_ALGORITHMS + ")", lastException);
+    }
+}

src/main/java/com/sparrowwallet/frigate/io/Storage.java

@@ -10,6 +10,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
+import java.util.zip.GZIPInputStream;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.PosixFilePermission;
@@ -23,32 +24,52 @@ public class Storage {
     public static final String FRIGATE_DIR = ".frigate";
     public static final String WINDOWS_FRIGATE_DIR = "Frigate";
 
-    public static File getSecp256k1ExtensionFile() {
+    public static final String UFSECP_EXTENSION_FILENAME = "ufsecp.duckdb_extension";
+
+    private static File ufsecpExtensionFile;
+
+    public static synchronized File getUfsecpExtensionFile() {
+        if(ufsecpExtensionFile == null) {
+            ufsecpExtensionFile = getExtensionFile(UFSECP_EXTENSION_FILENAME);
+        }
+        return ufsecpExtensionFile;
+    }
+
+    public static File getExtensionFile(String extensionFileName) {
         String resourcePath;
         String osName = System.getProperty("os.name");
         String osArch = System.getProperty("os.arch");
         if(osName.startsWith("Mac") && osArch.equals("aarch64")) {
-            resourcePath = "/native/macos/arm64/secp256k1.duckdb_extension";
+            resourcePath = "/native/macos/arm64/" + extensionFileName;
         } else if(osName.startsWith("Mac")) {
-            resourcePath = "/native/macos/amd64/secp256k1.duckdb_extension";
+            resourcePath = "/native/macos/amd64/" + extensionFileName;
         } else if(osName.startsWith("Windows")) {
-            resourcePath = "/native/windows/amd64/secp256k1.duckdb_extension";
+            resourcePath = "/native/windows/amd64/" + extensionFileName;
         } else if(osArch.equals("aarch64")) {
-            resourcePath = "/native/linux/arm64/secp256k1.duckdb_extension";
+            resourcePath = "/native/linux/arm64/" + extensionFileName;
         } else {
-            resourcePath = "/native/linux/amd64/secp256k1.duckdb_extension";
+            resourcePath = "/native/linux/amd64/" + extensionFileName;
         }
 
-        File extensionFile = new File(getFrigateDbDir(), "secp256k1.duckdb_extension");
+        File extensionFile = new File(getFrigateCacheDir(), extensionFileName);
 
-        try(InputStream is = Storage.class.getResourceAsStream(resourcePath)) {
-            if(is == null) {
-                throw new IOException("Could not find secp256k1 extension for the current platform: " + osName + " " + osArch);
+        try {
+            InputStream is = Storage.class.getResourceAsStream(resourcePath);
+            if(is != null) {
+                try(is) {
+                    Files.copy(is, extensionFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+                }
+            } else {
+                InputStream gzIs = Storage.class.getResourceAsStream(resourcePath + ".gz");
+                if(gzIs == null) {
+                    throw new IOException("Could not find " + extensionFileName + " for the current platform: " + osName + " " + osArch);
+                }
+                try(GZIPInputStream decompressed = new GZIPInputStream(gzIs)) {
+                    Files.copy(decompressed, extensionFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+                }
             }
-
-            Files.copy(is, extensionFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
         } catch(IOException e) {
-            log.error("Error loading secp256k1 extension", e);
+            log.error("Error loading " + extensionFileName, e);
         }
 
         return extensionFile;
@@ -62,6 +83,14 @@ public class Storage {
         return dbDir;
     }
 
+    public static File getFrigateCacheDir() {
+        File cacheDir = new File(getFrigateDir(), "cache");
+        if(!cacheDir.exists()) {
+            createOwnerOnlyDirectory(cacheDir);
+        }
+        return cacheDir;
+    }
+
     public static File getFrigateDir() {
         File frigateDir;
         Network network = Network.get();

src/main/java/module-info.java

@@ -1,21 +1,27 @@
 module com.sparrowwallet.frigate {
     requires com.sparrowwallet.drongo;
-    requires org.duckdb.duckdb_jdbc;
-    requires com.zaxxer.hikari;
+    requires duckdb.jdbc;
     requires com.fasterxml.jackson.annotation;
+    requires com.fasterxml.jackson.databind;
+    requires com.fasterxml.jackson.dataformat.toml;
     requires simple.json.rpc.core;
     requires simple.json.rpc.client;
     requires simple.json.rpc.server;
     requires com.google.gson;
     requires com.google.common;
     requires org.jcommander;
+    requires org.zeromq.jeromq;
     requires org.slf4j;
+    requires ch.qos.logback.core;
+    requires ch.qos.logback.classic;
     requires java.sql;
+    requires static java.desktop;
     exports com.sparrowwallet.frigate;
     exports com.sparrowwallet.frigate.io;
     exports com.sparrowwallet.frigate.bitcoind;
     exports com.sparrowwallet.frigate.electrum;
     exports com.sparrowwallet.frigate.index;
     exports com.sparrowwallet.frigate.cli;
-    opens com.sparrowwallet.frigate.io;
+    opens com.sparrowwallet.frigate.control to com.google.common;
+    opens com.sparrowwallet.frigate.io to com.fasterxml.jackson.databind;
 }

src/main/resources/config.toml.default

@@ -0,0 +1,36 @@
+# Frigate configuration
+# See https://github.com/sparrowwallet/frigate for documentation
+
+[core]
+connect = true
+# server = "http://127.0.0.1:8332"
+# authType = "COOKIE"            # COOKIE or USERPASS
+# dataDir = "/home/bitcoin/.bitcoin"
+# auth = "user:password"         # only needed for USERPASS
+# zmqSequenceEndpoint = "tcp://127.0.0.1:28336"   # bitcoind -zmqpubsequence endpoint for low-latency mempool ingestion
+# rpcRequestTimeoutSeconds = 60                   # per-RPC read timeout (raise on slow/remote bitcoind, lower on fast LAN)
+# rpcBatchSize = 100                              # max sub-requests per JSON-RPC array batch (mempool fill)
+
+[index]
+# startHeight = 0                # default: 709632 on mainnet (Taproot activation), 0 on testnet
+# cacheSize = "10M"              # scriptPubKey cache entries (default: 10M, ~4GB RAM)
+
+[scan]
+# batchSize = 300000             # rows per GPU dispatch (reduce if scanning hangs on older GPUs)
+# computeBackend = "AUTO"        # AUTO, GPU, or CPU (AUTO prefers GPU over CPU)
+# dbThreads = 4                  # limit DuckDB threads (reduces CPU load when computeBackend = "CPU")
+# memoryLimit = "8GB"            # cap DuckDB memory usage (default: 80% of system RAM)
+# maxLabels = 10                 # maximum number of labels accepted per silent payments subscription
+# maxSubscriptions = 100         # maximum number of silent payments subscriptions per connection
+
+[server]
+# host = "ssl://xyz.com:50002"   # advertised in server.features; use array for multiple. Omit to advertise nothing.
+# tcp = "tcp://0.0.0.0:50001"    # plaintext listener bind URL; omit (or set to "") to disable. Default if neither tcp nor ssl is set.
+# ssl = "ssl://0.0.0.0:50002"    # SSL listener bind URL; omit to disable
+# sslCert = "cert.pem"           # PEM certificate (chain allowed); bare filename resolves under Frigate's home dir, or use an absolute path
+# sslKey  = "key.pem"            # PEM-encoded PKCS#8 private key; bare filename resolves under Frigate's home dir, or use an absolute path
+# backendElectrumServer = "tcp://localhost:60001"   # backend must listen on a port distinct from Frigate's tcp/ssl listeners above
+
+# [database]
+# url = "jdbc:duckdb:/custom/path/frigate.duckdb"
+# readUrls = ["jdbc:duckdb:/replica1/frigate.duckdb"]

src/main/resources/logback.xml

@@ -1,11 +1,10 @@
 <configuration>
     <statusListener class="ch.qos.logback.core.status.NopStatusListener" />
 
+    <turboFilter class="com.sparrowwallet.frigate.io.JsonRpcErrorFilter" />
+
     <logger name="sun.net.www.protocol.http.HttpURLConnection" level="INFO" />
     <logger name="com.github.arteam.simplejsonrpc.server.JsonRpcServer" level="INFO" />
-    <logger name="com.zaxxer.hikari.HikariDataSource" level="WARN" />
-    <logger name="com.zaxxer.hikari.pool.PoolBase" level="ERROR" />
-    <logger name="com.zaxxer.hikari.pool.HikariPool" level="ERROR" />
 
     <define name="appDir" class="com.sparrowwallet.drongo.PropertyDefiner">
         <application>frigate</application>
@@ -14,7 +13,7 @@
     <appender name="FILE" class="ch.qos.logback.core.FileAppender">
         <file>${appDir}/frigate.log</file>
         <encoder>
-            <pattern>%date %level [%thread] %logger{10} [%file:%line] %msg%n</pattern>
+            <pattern>%date %level [%thread] %logger{10} %msg%n</pattern>
         </encoder>
     </appender>
 

src/test/java/com/sparrowwallet/frigate/bitcoind/VerboseBlockTest.java

@@ -0,0 +1,128 @@
+package com.sparrowwallet.frigate.bitcoind;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class VerboseBlockTest {
+    private static final String V3_JSON = """
+            {
+              "hash": "0000000000000000000111111111111111111111111111111111111111111111",
+              "confirmations": 1,
+              "height": 800000,
+              "time": 1690000000,
+              "previousblockhash": "0000000000000000000222222222222222222222222222222222222222222222",
+              "tx": [
+                {
+                  "txid": "aaaa000000000000000000000000000000000000000000000000000000000000",
+                  "hash": "aaaa000000000000000000000000000000000000000000000000000000000000",
+                  "hex": "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff03520101ffffffff0100000000000000000000000000",
+                  "vin": [
+                    {
+                      "coinbase": "520101",
+                      "sequence": 4294967295
+                    }
+                  ],
+                  "vout": [
+                    {
+                      "value": 0.0,
+                      "n": 0,
+                      "scriptPubKey": {
+                        "asm": "",
+                        "hex": "",
+                        "type": "nonstandard"
+                      }
+                    }
+                  ]
+                },
+                {
+                  "txid": "bbbb000000000000000000000000000000000000000000000000000000000000",
+                  "hex": "0200000001cccc00000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000002251200000000000000000000000000000000000000000000000000000000000000abc00000000",
+                  "vin": [
+                    {
+                      "txid": "cccc000000000000000000000000000000000000000000000000000000000000",
+                      "vout": 0,
+                      "scriptSig": {"asm": "", "hex": ""},
+                      "sequence": 4294967295,
+                      "prevout": {
+                        "generated": false,
+                        "height": 799999,
+                        "value": 0.001,
+                        "scriptPubKey": {
+                          "asm": "OP_1 abcd",
+                          "hex": "5120000000000000000000000000000000000000000000000000000000000000abcd",
+                          "address": "bc1pdummy",
+                          "type": "witness_v1_taproot"
+                        }
+                      }
+                    }
+                  ],
+                  "vout": [
+                    {
+                      "value": 0.0009,
+                      "n": 0,
+                      "scriptPubKey": {
+                        "hex": "5120000000000000000000000000000000000000000000000000000000000000abc0",
+                        "type": "witness_v1_taproot"
+                      }
+                    }
+                  ]
+                }
+              ]
+            }
+            """;
+
+    @Test
+    public void deserialisesV3Block() throws Exception {
+        ObjectMapper mapper = new ObjectMapper();
+        VerboseBlock vb = mapper.readValue(V3_JSON, VerboseBlock.class);
+
+        assertEquals("0000000000000000000111111111111111111111111111111111111111111111", vb.hash());
+        assertEquals(800000, vb.height());
+        assertEquals(1690000000L, vb.time());
+        assertEquals(2, vb.tx().size());
+    }
+
+    @Test
+    public void coinbaseInputHasNoTxid() throws Exception {
+        ObjectMapper mapper = new ObjectMapper();
+        VerboseBlock vb = mapper.readValue(V3_JSON, VerboseBlock.class);
+
+        VerboseBlock.VerboseTransaction coinbase = vb.tx().get(0);
+        VerboseBlock.VerboseVin vin = coinbase.vin().get(0);
+        assertTrue(vin.isCoinbase());
+        assertNull(vin.txid());
+        assertNull(vin.prevout());
+    }
+
+    @Test
+    public void nonCoinbaseInputCarriesPrevoutScriptPubKey() throws Exception {
+        ObjectMapper mapper = new ObjectMapper();
+        VerboseBlock vb = mapper.readValue(V3_JSON, VerboseBlock.class);
+
+        VerboseBlock.VerboseTransaction spending = vb.tx().get(1);
+        VerboseBlock.VerboseVin vin = spending.vin().get(0);
+        assertFalse(vin.isCoinbase());
+        assertEquals("cccc000000000000000000000000000000000000000000000000000000000000", vin.txid());
+        assertEquals(0, vin.voutIndex().intValue());
+        assertNotNull(vin.prevout());
+        assertNotNull(vin.prevout().scriptPubKey());
+        assertEquals("5120000000000000000000000000000000000000000000000000000000000000abcd", vin.prevout().scriptPubKey().hex());
+    }
+
+    @Test
+    public void outputCarriesScriptPubKey() throws Exception {
+        ObjectMapper mapper = new ObjectMapper();
+        VerboseBlock vb = mapper.readValue(V3_JSON, VerboseBlock.class);
+
+        VerboseBlock.VerboseTransaction spending = vb.tx().get(1);
+        VerboseBlock.VerboseVout vout = spending.vout().get(0);
+        assertEquals(0, vout.n());
+        assertEquals("5120000000000000000000000000000000000000000000000000000000000000abc0", vout.scriptPubKey().hex());
+    }
+}

src/test/java/com/sparrowwallet/frigate/electrum/AdvertisedHostsTest.java

@@ -0,0 +1,43 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.sparrowwallet.frigate.io.Server;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+public class AdvertisedHostsTest {
+    @Test
+    public void emptyInputYieldsEmptyMap() {
+        Map<String, ServerFeatures.HostInfo> hosts = ElectrumServerService.buildAdvertisedHosts(List.of());
+        Assertions.assertTrue(hosts.isEmpty());
+    }
+
+    @Test
+    public void tcpAndSslOnSameHostMerge() {
+        Map<String, ServerFeatures.HostInfo> hosts = ElectrumServerService.buildAdvertisedHosts(List.of(Server.fromString("tcp://example.com:50001"), Server.fromString("ssl://example.com:50002")));
+        Assertions.assertEquals(1, hosts.size());
+        ServerFeatures.HostInfo info = hosts.get("example.com");
+        Assertions.assertEquals(50001, info.tcp_port());
+        Assertions.assertEquals(50002, info.ssl_port());
+    }
+
+    @Test
+    public void differentHostsRemainSeparate() {
+        Map<String, ServerFeatures.HostInfo> hosts = ElectrumServerService.buildAdvertisedHosts(List.of(Server.fromString("tcp://example.com:50001"), Server.fromString("ssl://onion.example:443")));
+        Assertions.assertEquals(2, hosts.size());
+        Assertions.assertEquals(50001, hosts.get("example.com").tcp_port());
+        Assertions.assertNull(hosts.get("example.com").ssl_port());
+        Assertions.assertEquals(443, hosts.get("onion.example").ssl_port());
+        Assertions.assertNull(hosts.get("onion.example").tcp_port());
+    }
+
+    @Test
+    public void singleProtocolLeavesOtherNull() {
+        Map<String, ServerFeatures.HostInfo> hosts = ElectrumServerService.buildAdvertisedHosts(List.of(Server.fromString("ssl://example.com:443")));
+        ServerFeatures.HostInfo info = hosts.get("example.com");
+        Assertions.assertNull(info.tcp_port());
+        Assertions.assertEquals(443, info.ssl_port());
+    }
+}

src/test/java/com/sparrowwallet/frigate/electrum/ServerFeaturesTest.java

@@ -0,0 +1,33 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class ServerFeaturesTest {
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+
+    @Test
+    public void hostInfoDeserializesObjectShape() throws Exception {
+        ServerFeatures.HostInfo info = MAPPER.readValue("{\"tcp_port\":50001,\"ssl_port\":50002}", ServerFeatures.HostInfo.class);
+        Assertions.assertEquals(50001, info.tcp_port());
+        Assertions.assertEquals(50002, info.ssl_port());
+    }
+
+    @Test
+    public void hostInfoDeserializesBareIntegerAsTcpPort() throws Exception {
+        ServerFeatures.HostInfo info = MAPPER.readValue("40001", ServerFeatures.HostInfo.class);
+        Assertions.assertEquals(40001, info.tcp_port());
+        Assertions.assertNull(info.ssl_port());
+    }
+
+    @Test
+    public void serverFeaturesDeserializesMixedHostsMap() throws Exception {
+        String json = "{\"hosts\":{\"a.example.com\":{\"tcp_port\":50001,\"ssl_port\":50002},\"b.example.com\":40001}}";
+        ServerFeatures features = MAPPER.readValue(json, ServerFeatures.class);
+        Assertions.assertEquals(50001, features.hosts().get("a.example.com").tcp_port());
+        Assertions.assertEquals(50002, features.hosts().get("a.example.com").ssl_port());
+        Assertions.assertEquals(40001, features.hosts().get("b.example.com").tcp_port());
+        Assertions.assertNull(features.hosts().get("b.example.com").ssl_port());
+    }
+}

src/test/java/com/sparrowwallet/frigate/electrum/SilentPaymentsTxEntryTest.java

@@ -0,0 +1,24 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class SilentPaymentsTxEntryTest {
+    @Test
+    public void serializedFormHasNoFeeField() throws Exception {
+        SilentPaymentsTxEntry entry = new SilentPaymentsTxEntry(800000, "abcd", "0011");
+        String json = new ObjectMapper().writeValueAsString(entry);
+        Assertions.assertFalse(json.contains("\"fee\""));
+        Assertions.assertTrue(json.contains("\"tweak_key\""));
+        Assertions.assertTrue(json.contains("\"tx_hash\""));
+        Assertions.assertTrue(json.contains("\"height\""));
+    }
+
+    @Test
+    public void nullTweakKeyIsOmitted() throws Exception {
+        SilentPaymentsTxEntry entry = new SilentPaymentsTxEntry(800000, "abcd", null);
+        String json = new ObjectMapper().writeValueAsString(entry);
+        Assertions.assertFalse(json.contains("\"tweak_key\""));
+    }
+}

src/test/java/com/sparrowwallet/frigate/electrum/TxEntryTest.java

@@ -0,0 +1,22 @@
+package com.sparrowwallet.frigate.electrum;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TxEntryTest {
+    @Test
+    public void scripthashEntryHasNoTweakKeyField() throws Exception {
+        TxEntry entry = new TxEntry(800000, 0, "abcd");
+        String json = new ObjectMapper().writeValueAsString(entry);
+        Assertions.assertFalse(json.contains("\"tweak_key\""));
+        Assertions.assertTrue(json.contains("\"tx_hash\""));
+    }
+
+    @Test
+    public void nullFeeIsOmitted() throws Exception {
+        TxEntry entry = new TxEntry(800000, 0, "abcd");
+        String json = new ObjectMapper().writeValueAsString(entry);
+        Assertions.assertFalse(json.contains("\"fee\""));
+    }
+}

src/test/java/com/sparrowwallet/frigate/index/HistoricalScanMetricsTest.java

@@ -0,0 +1,147 @@
+package com.sparrowwallet.frigate.index;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class HistoricalScanMetricsTest {
+    @Test
+    public void resultBucketBoundaries() {
+        assertEquals(0, HistoricalScanMetrics.resultBucket(0));
+        assertEquals(1, HistoricalScanMetrics.resultBucket(1));
+        assertEquals(1, HistoricalScanMetrics.resultBucket(10));
+        assertEquals(2, HistoricalScanMetrics.resultBucket(11));
+        assertEquals(2, HistoricalScanMetrics.resultBucket(100));
+        assertEquals(3, HistoricalScanMetrics.resultBucket(101));
+        assertEquals(3, HistoricalScanMetrics.resultBucket(1000));
+        assertEquals(4, HistoricalScanMetrics.resultBucket(1001));
+        assertEquals(4, HistoricalScanMetrics.resultBucket(10000));
+        assertEquals(5, HistoricalScanMetrics.resultBucket(10001));
+    }
+
+    //pair each sample input with the label the log line will use, so a future edit that changes a boundary without updating
+    //the corresponding label (or vice versa) fails here instead of silently producing a misleading aggregate line.
+    @Test
+    public void resultLabelsMatchBoundaries() {
+        assertEquals("0", HistoricalScanMetrics.RESULT_LABELS[HistoricalScanMetrics.resultBucket(0)]);
+        assertEquals("1-10", HistoricalScanMetrics.RESULT_LABELS[HistoricalScanMetrics.resultBucket(5)]);
+        assertEquals("11-100", HistoricalScanMetrics.RESULT_LABELS[HistoricalScanMetrics.resultBucket(50)]);
+        assertEquals("101-1000", HistoricalScanMetrics.RESULT_LABELS[HistoricalScanMetrics.resultBucket(500)]);
+        assertEquals("1001-10000", HistoricalScanMetrics.RESULT_LABELS[HistoricalScanMetrics.resultBucket(5000)]);
+        assertEquals("10001+", HistoricalScanMetrics.RESULT_LABELS[HistoricalScanMetrics.resultBucket(50000)]);
+    }
+
+    @Test
+    public void durationLabelsMatchBoundaries() {
+        assertEquals("0-100ms", HistoricalScanMetrics.DURATION_LABELS[HistoricalScanMetrics.durationBucket(50)]);
+        assertEquals("100-500ms", HistoricalScanMetrics.DURATION_LABELS[HistoricalScanMetrics.durationBucket(250)]);
+        assertEquals("500ms-2s", HistoricalScanMetrics.DURATION_LABELS[HistoricalScanMetrics.durationBucket(1000)]);
+        assertEquals("2-10s", HistoricalScanMetrics.DURATION_LABELS[HistoricalScanMetrics.durationBucket(5000)]);
+        assertEquals("10-60s", HistoricalScanMetrics.DURATION_LABELS[HistoricalScanMetrics.durationBucket(30000)]);
+        assertEquals("60s+", HistoricalScanMetrics.DURATION_LABELS[HistoricalScanMetrics.durationBucket(120000)]);
+    }
+
+    @Test
+    public void durationBucketBoundaries() {
+        assertEquals(0, HistoricalScanMetrics.durationBucket(0));
+        assertEquals(0, HistoricalScanMetrics.durationBucket(99));
+        assertEquals(1, HistoricalScanMetrics.durationBucket(100));
+        assertEquals(1, HistoricalScanMetrics.durationBucket(499));
+        assertEquals(2, HistoricalScanMetrics.durationBucket(500));
+        assertEquals(2, HistoricalScanMetrics.durationBucket(1999));
+        assertEquals(3, HistoricalScanMetrics.durationBucket(2000));
+        assertEquals(3, HistoricalScanMetrics.durationBucket(9999));
+        assertEquals(4, HistoricalScanMetrics.durationBucket(10000));
+        assertEquals(4, HistoricalScanMetrics.durationBucket(59999));
+        assertEquals(5, HistoricalScanMetrics.durationBucket(60000));
+    }
+
+    @Test
+    public void thresholdSuppressesBelowTen() {
+        HistoricalScanMetrics m = new HistoricalScanMetrics();
+        for(int i = 0; i < 9; i++) {
+            m.record(5, 200);
+        }
+        assertTrue(m.format(m.snapshotAndReset()).isEmpty());
+    }
+
+    @Test
+    public void exactlyTenEmitsAsTen() {
+        HistoricalScanMetrics m = new HistoricalScanMetrics();
+        for(int i = 0; i < 10; i++) {
+            m.record(5, 200);
+        }
+        String line = m.format(m.snapshotAndReset()).orElseThrow();
+        assertTrue(line.contains("1-10:10"), line);
+        assertTrue(line.contains("100-500ms:10"), line);
+    }
+
+    @Test
+    public void fourteenRoundsDownToTen() {
+        HistoricalScanMetrics m = new HistoricalScanMetrics();
+        for(int i = 0; i < 14; i++) {
+            m.record(5, 200);
+        }
+        assertTrue(m.format(m.snapshotAndReset()).orElseThrow().contains("1-10:10"));
+    }
+
+    @Test
+    public void fifteenRoundsUpToTwenty() {
+        HistoricalScanMetrics m = new HistoricalScanMetrics();
+        for(int i = 0; i < 15; i++) {
+            m.record(5, 200);
+        }
+        assertTrue(m.format(m.snapshotAndReset()).orElseThrow().contains("1-10:20"));
+    }
+
+    @Test
+    public void snapshotResetsCounters() {
+        HistoricalScanMetrics m = new HistoricalScanMetrics();
+        for(int i = 0; i < 20; i++) {
+            m.record(5, 200);
+        }
+        m.snapshotAndReset();
+        assertTrue(m.format(m.snapshotAndReset()).isEmpty());
+    }
+
+    //spread result counts across all five non-zero buckets so no result bucket reaches threshold, while every duration lands in
+    //the same bucket so it does — verifies one-sided emission.
+    @Test
+    public void onlyOneHistogramEmittedWhenOnlyOneClearsThreshold() {
+        HistoricalScanMetrics m = new HistoricalScanMetrics();
+        int[] resultsPerBucket = {0, 0, 5, 5, 50, 50, 500, 500, 5000, 5000};
+        for(int n : resultsPerBucket) {
+            m.record(n, 200);
+        }
+        String line = m.format(m.snapshotAndReset()).orElseThrow();
+        assertFalse(line.contains("results ["), line);
+        assertTrue(line.contains("duration ["), line);
+    }
+
+    @Test
+    public void emptyWhenNoRecords() {
+        HistoricalScanMetrics m = new HistoricalScanMetrics();
+        assertTrue(m.format(m.snapshotAndReset()).isEmpty());
+    }
+
+    //mechanical guardrail: future edits to the format must not introduce summary statistics, relative timestamps, or absolute
+    //ISO timestamps. The forbidden tokens are bare (no colons) so "last value: 123ms" or "min 100ms" both trip the check.
+    @Test
+    public void formattedLineHasNoForbiddenSubstrings() {
+        HistoricalScanMetrics m = new HistoricalScanMetrics();
+        for(int i = 0; i < 20; i++) {
+            m.record(50, 1500);
+        }
+        String line = m.format(m.snapshotAndReset()).orElseThrow();
+        String lower = line.toLowerCase();
+        for(String forbidden : new String[]{"min", "max", "avg", "mean", "last", "ago"}) {
+            assertFalse(lower.contains(forbidden), "line must not contain '" + forbidden + "': " + line);
+        }
+        assertFalse(line.matches(".*\\d{4}-\\d{2}-\\d{2}.*"), "line must not contain ISO date: " + line);
+        assertFalse(line.matches(".*\\d{2}:\\d{2}:\\d{2}.*"), "line must not contain ISO time: " + line);
+    }
+}

src/test/java/com/sparrowwallet/frigate/index/SingleDbManagerTest.java

@@ -0,0 +1,63 @@
+package com.sparrowwallet.frigate.index;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class SingleDbManagerTest {
+    //regression for the three-way DB deadlock: once a writer holds the writerWaiting permit, any new executeRead parks in wait().
+    //The implicit close() in try-with-resources does shutdown+awaitTermination, which doesn't interrupt - shutdownNow is the load-bearing call.
+    @Test
+    public void testShutdownNowInterruptsExecuteReadParkedBehindWriter() throws Exception {
+        SingleDbManager dbManager = new SingleDbManager(DbManager.DB_PREFIX);
+        CountDownLatch writerHoldingPermit = new CountDownLatch(1);
+        CountDownLatch writerMayRelease = new CountDownLatch(1);
+
+        //writer takes the writerWaiting permit + write lock and holds them until the test releases it
+        Thread writer = new Thread(() -> {
+            try {
+                dbManager.executeWrite(conn -> {
+                    writerHoldingPermit.countDown();
+                    try {
+                        writerMayRelease.await();
+                    } catch(InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                    }
+                    return null;
+                });
+            } catch(Exception ignored) {
+            }
+        }, "deadlock-test-writer");
+
+        try {
+            writer.start();
+            writerHoldingPermit.await();
+
+            //schedule a task that will park in SingleDbManager.executeRead.wait() (writerWaiting.availablePermits()==0)
+            ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
+            executor.scheduleAtFixedRate(() -> {
+                try {
+                    dbManager.executeRead(conn -> null);
+                } catch(Exception ignored) {
+                }
+            }, 0, 100, TimeUnit.MILLISECONDS);
+
+            //give the task time to fire and park
+            Thread.sleep(200);
+
+            //the load-bearing claim of the fix: shutdownNow interrupts the parked executeRead so the executor terminates promptly
+            executor.shutdownNow();
+            boolean terminated = executor.awaitTermination(2, TimeUnit.SECONDS);
+
+            assertTrue(terminated, "shutdownNow should interrupt the parked executeRead and let the executor terminate");
+        } finally {
+            writerMayRelease.countDown();
+            writer.join(2000);
+            dbManager.close();
+        }
+    }
+}

src/test/java/com/sparrowwallet/frigate/io/JsonRpcErrorFilterTest.java

@@ -0,0 +1,48 @@
+package com.sparrowwallet.frigate.io;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.core.spi.FilterReply;
+import com.github.arteam.simplejsonrpc.core.annotation.JsonRpcError;
+import org.junit.jupiter.api.Test;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class JsonRpcErrorFilterTest {
+    @Test
+    public void filterRegisteredFromLogbackConfig() {
+        LoggerContext ctx = (LoggerContext) LoggerFactory.getILoggerFactory();
+        assertTrue(ctx.getTurboFilterList().stream().anyMatch(f -> f instanceof JsonRpcErrorFilter), "JsonRpcErrorFilter must be loaded from logback.xml");
+    }
+
+    @Test
+    public void deniesAnnotatedRootCause() {
+        JsonRpcErrorFilter filter = new JsonRpcErrorFilter();
+        Logger logger = (Logger) LoggerFactory.getLogger("com.github.arteam.simplejsonrpc.server.JsonRpcServer");
+        Throwable cause = new AnnotatedException();
+        Throwable wrapped = new RuntimeException(cause);
+        assertEquals(FilterReply.DENY, filter.decide(null, logger, Level.ERROR, "Error while processing", null, wrapped));
+    }
+
+    @Test
+    public void neutralWhenNoAnnotation() {
+        JsonRpcErrorFilter filter = new JsonRpcErrorFilter();
+        Logger logger = (Logger) LoggerFactory.getLogger("com.github.arteam.simplejsonrpc.server.JsonRpcServer");
+        Throwable wrapped = new RuntimeException(new IllegalStateException("boom"));
+        assertEquals(FilterReply.NEUTRAL, filter.decide(null, logger, Level.ERROR, "Error while processing", null, wrapped));
+    }
+
+    @Test
+    public void neutralForOtherLoggers() {
+        JsonRpcErrorFilter filter = new JsonRpcErrorFilter();
+        Logger logger = (Logger) LoggerFactory.getLogger("some.other.Logger");
+        assertEquals(FilterReply.NEUTRAL, filter.decide(null, logger, Level.ERROR, "msg", null, new AnnotatedException()));
+    }
+
+    @JsonRpcError(code = -1, message = "test")
+    private static class AnnotatedException extends Exception {
+    }
+}

src/test/java/com/sparrowwallet/frigate/io/ServerConfigHostTest.java

@@ -0,0 +1,103 @@
+package com.sparrowwallet.frigate.io;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.dataformat.toml.TomlMapper;
+import com.sparrowwallet.frigate.ConfigurationException;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+public class ServerConfigHostTest {
+    private static final TomlMapper MAPPER = TomlMapper.builder().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).build();
+
+    @Test
+    public void unsetHostAdvertisesNothing() throws Exception {
+        Config.ServerConfig config = parse("");
+        Assertions.assertTrue(config.getAdvertisedHosts().isEmpty());
+    }
+
+    @Test
+    public void emptyStringHostAdvertisesNothing() throws Exception {
+        Config.ServerConfig config = parse("host = \"\"");
+        Assertions.assertTrue(config.getAdvertisedHosts().isEmpty());
+    }
+
+    @Test
+    public void bareHostnameExpandsToBindPorts() throws Exception {
+        Config.ServerConfig config = parse("""
+                host = "example.com"
+                tcp = "tcp://0.0.0.0:50001"
+                ssl = "ssl://0.0.0.0:50002"
+                """);
+        List<Server> advertised = config.getAdvertisedHosts();
+        Assertions.assertEquals(2, advertised.size());
+        Assertions.assertEquals("tcp://example.com:50001", advertised.get(0).getUrl());
+        Assertions.assertEquals("ssl://example.com:50002", advertised.get(1).getUrl());
+    }
+
+    @Test
+    public void bareHostnameWithOnlyTcpBindAdvertisesTcpOnly() throws Exception {
+        Config.ServerConfig config = parse("""
+                host = "example.com"
+                tcp = "tcp://0.0.0.0:50001"
+                """);
+        List<Server> advertised = config.getAdvertisedHosts();
+        Assertions.assertEquals(1, advertised.size());
+        Assertions.assertEquals(Protocol.TCP, advertised.getFirst().getProtocol());
+        Assertions.assertEquals(50001, advertised.getFirst().getHostAndPort().getPort());
+    }
+
+    @Test
+    public void urlFormUsesExplicitPort() throws Exception {
+        Config.ServerConfig config = parse("""
+                host = "ssl://example.com:443"
+                tcp = "tcp://0.0.0.0:50001"
+                """);
+        List<Server> advertised = config.getAdvertisedHosts();
+        Assertions.assertEquals(1, advertised.size());
+        Assertions.assertEquals(Protocol.SSL, advertised.getFirst().getProtocol());
+        Assertions.assertEquals(443, advertised.getFirst().getHostAndPort().getPort());
+    }
+
+    @Test
+    public void arrayFormAdvertisesEachEntry() throws Exception {
+        Config.ServerConfig config = parse("""
+                host = ["tcp://example.com:80", "ssl://example.com:443"]
+                """);
+        List<Server> advertised = config.getAdvertisedHosts();
+        Assertions.assertEquals(2, advertised.size());
+        Assertions.assertEquals("tcp://example.com:80", advertised.get(0).getUrl());
+        Assertions.assertEquals("ssl://example.com:443", advertised.get(1).getUrl());
+    }
+
+    @Test
+    public void mixedArrayAcceptsBareAndUrl() throws Exception {
+        Config.ServerConfig config = parse("""
+                host = ["example.com", "ssl://onion.example:443"]
+                tcp = "tcp://0.0.0.0:50001"
+                """);
+        List<Server> advertised = config.getAdvertisedHosts();
+        Assertions.assertEquals(2, advertised.size());
+        Assertions.assertEquals("tcp://example.com:50001", advertised.get(0).getUrl());
+        Assertions.assertEquals("ssl://onion.example:443", advertised.get(1).getUrl());
+    }
+
+    @Test
+    public void httpSchemeIsRejected() throws Exception {
+        Config.ServerConfig config = parse("host = \"http://example.com:80\"");
+        Assertions.assertThrows(ConfigurationException.class, config::getAdvertisedHosts);
+    }
+
+    @Test
+    public void urlWithoutPortIsRejected() throws Exception {
+        Config.ServerConfig config = parse("host = \"tcp://example.com\"");
+        Assertions.assertThrows(ConfigurationException.class, config::getAdvertisedHosts);
+    }
+
+    private static Config.ServerConfig parse(String tomlBody) throws Exception {
+        String toml = "[server]\n" + tomlBody;
+        Config config = MAPPER.readValue(toml, Config.class);
+        return config.getServer();
+    }
+}