Adapt Dockerfile for BTCPay deployment

Changelog-None

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+> [!IMPORTANT]
+>
+> 25.09 FREEZE July 28TH: Non-bugfix PRs not ready by this date will wait for 25.12.
+>
+> RC1 is scheduled on _August 11th_
+>
+> The final release is scheduled for September 1st.
+
+
+## Checklist
+Before submitting the PR, ensure the following tasks are completed. If an item is not applicable to your PR, please mark it as checked:
+
+- [ ] The changelog has been updated in the relevant commit(s) according to the [guidelines](https://docs.corelightning.org/docs/coding-style-guidelines#changelog-entries-in-commit-messages).
+- [ ] Tests have been added or modified to reflect the changes.
+- [ ] Documentation has been reviewed and updated as needed.
+- [ ] Related issues have been listed and linked, including any that this PR closes.

.github/scripts/install-bitcoind.sh

@@ -2,6 +2,9 @@
 
 set -e
 
+export BITCOIN_VERSION=27.1
+export ELEMENTS_VERSION=23.2.1
+
 DIRNAME="bitcoin-${BITCOIN_VERSION}"
 EDIRNAME="elements-${ELEMENTS_VERSION}"
 FILENAME="${DIRNAME}-x86_64-linux-gnu.tar.gz"

.github/scripts/setup.sh

@@ -1,8 +1,6 @@
 #!/bin/bash
 set -e
 export DEBIAN_FRONTEND=noninteractive
-export BITCOIN_VERSION=26.1
-export ELEMENTS_VERSION=22.0.2
 export RUST_VERSION=stable
 
 sudo useradd -ms /bin/bash tester
@@ -22,11 +20,18 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      gcc-arm-none-eabi \
      gettext \
      git \
+     gnupg \
+     jq \
      libc6-dev-arm64-cross \
      libc6-dev-armhf-cross \
+     libev-dev \
+     libevent-dev \
+     libffi-dev \
+     libicu-dev \
      libpq-dev \
      libprotobuf-c-dev \
      libsqlite3-dev \
+     libssl-dev \
      libtool \
      libxml2-utils \
      locales \
@@ -44,29 +49,18 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      software-properties-common \
      sudo \
      tcl \
+     tclsh \
      unzip \
      valgrind \
      wget \
      xsltproc \
+     systemtap-sdt-dev \
      zlib1g-dev
 
 echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
 sudo chmod 0440 /etc/sudoers.d/tester
 
-(
-    cd /tmp/ || exit 1
-    wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
-    wget https://github.com/ElementsProject/elements/releases/download/elements-${ELEMENTS_VERSION}/elements-${ELEMENTS_VERSION}-x86_64-linux-gnu.tar.gz
-    tar -xf bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
-    tar -xf elements-${ELEMENTS_VERSION}-x86_64-linux-gnu.tar.gz
-    sudo mv bitcoin-${BITCOIN_VERSION}/bin/* /usr/local/bin
-    sudo mv elements-${ELEMENTS_VERSION}/bin/* /usr/local/bin
-    rm -rf \
-       bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz \
-       bitcoin-${BITCOIN_VERSION} \
-       elements-${ELEMENTS_VERSION}-x86_64-linux-gnu.tar.gz \
-       elements-${ELEMENTS_VERSION}
-)
+"$(dirname "$0")"/install-bitcoind.sh
 
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
      -y --default-toolchain ${RUST_VERSION}
@@ -78,20 +72,17 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
 # which comes from the same tree.  Makes sense!
 
 # And
-#   grpcio-tools-1.54.0` requires `protobuf = ">=4.21.6,<5.0dev"`
+#   grpcio-tools-1.69.0` requires `protobuf = ">=5.26.1,<6.0dev"`
 
 # Now, protoc changed to date-based releases, BUT Python protobuf
 # didn't, so Python protobuf 4.21.12 (in Ubuntu 23.04) corresponds to
 # protoc 21.12 (which, FYI, is packaged in Ubuntu as version 3.21.12).
 
-# So we're going to nail these versions as 21.12, which is what recent
-# Ubuntu has, and hopefully everyone else can get.  And this means that
-# When CI checks that no files have changed under regeneration, you won't
-# get a fail just because the dev's protoc is a different version.
+# In general protobuf version x.y.z corresponds to protoc version y.z
 
 # Honorable mention go to Matt Whitlock for spelunking this horror with me!
 
-PROTOC_VERSION=21.12
+PROTOC_VERSION=29.4
 PB_REL="https://github.com/protocolbuffers/protobuf/releases"
 curl -LO $PB_REL/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip
 sudo unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local/

.github/scripts/sync-rpc-cmds.py

@@ -2,26 +2,30 @@ import os
 from time import sleep
 import requests
 import re
+from enum import Enum
 
 # readme url
-URL = "https://dash.readme.com/api/v1/docs"
+URL = "https://dash.readme.com/api/v1"
 # category id for API reference
 CATEGORY_ID = "63e4e160c60b2e001dd1cc4e"
+CATEGORY_SLUG = "json-rpc-apis"
 
 
-def checkIfDocIsPresent(title, headers):
+class Action(Enum):
+    ADD = 'add'
+    UPDATE = 'update'
+    DELETE = 'delete'
 
-    check_url = URL + "/" + title
-    response = requests.get(check_url, headers=headers)
 
+def getListOfRPCDocs(headers):
+    response = requests.get(f"{URL}/categories/{CATEGORY_SLUG}/docs", headers=headers)
     if response.status_code == 200:
-        return True
+        return response.json()
     else:
-        return False
+        return []
 
 
-def publishDoc(title, body, order):
-    key = os.environ.get("README_API_KEY")
+def publishDoc(action, title, body, order, headers):
     payload = {
         "title": title,
         "type": "basic",
@@ -30,28 +34,30 @@ def publishDoc(title, body, order):
         "hidden": False,
         "order": order,
     }
-    headers = {
-        "accept": "application/json",
-        "content-type": "application/json",
-        "authorization": "Basic " + key,
-    }
-
-    isDocPresent = checkIfDocIsPresent(title, headers)
-    if isDocPresent:
-        # update doc
-        update_url = URL + "/" + title  # title == slug
-        response = requests.put(update_url, json=payload, headers=headers)
-        if response.status_code != 200:
-            print(response.text)
-        else:
-            print("Updated ", title)
-    else:
+    # title == slug
+    if action == Action.ADD:
         # create doc
-        response = requests.post(URL, json=payload, headers=headers)
+        response = requests.post(URL + "/docs", json=payload, headers=headers)
         if response.status_code != 201:
             print(response.text)
         else:
             print("Created ", title)
+    elif action == Action.UPDATE:
+        # update doc
+        response = requests.put(f"{URL}/docs/{title}", json=payload, headers=headers)
+        if response.status_code != 200:
+            print(response.text)
+        else:
+            print("Updated ", title)
+    elif action == Action.DELETE:
+        # delete doc
+        response = requests.delete(f"{URL}/docs/{title}", headers=headers)
+        if response.status_code != 204:
+            print(response.text)
+        else:
+            print("Deleted ", title)
+    else:
+        print("Invalid action")
 
 
 def extract_rpc_commands(rst_content):
@@ -69,19 +75,36 @@ def extract_rpc_commands(rst_content):
 
 
 def main():
+    # define headers for requests
+    headers = {
+        "accept": "application/json",
+        "content-type": "application/json",
+        "authorization": "Basic " + os.environ.get("README_API_KEY"),
+    }
+
     # path to the rst file from where we fetch all the RPC commands
     path_to_rst = "doc/index.rst"
     with open(path_to_rst, "r") as file:
         rst_content = file.read()
 
-    commands = extract_rpc_commands(rst_content)
-    if commands:
+    commands_from_local = extract_rpc_commands(rst_content)
+    commands_from_readme = getListOfRPCDocs(headers)
+
+    # Compare local and server commands list to get the list of command to add or delete
+    commands_local_title = set(command[0] for command in commands_from_local)
+    commands_readme_title = set(command['title'] for command in commands_from_readme)
+    commands_to_delete = commands_readme_title - commands_local_title
+    commands_to_add = commands_local_title - commands_readme_title
+    for name in commands_to_delete:
+        publishDoc(Action.DELETE, name, "", 0, headers)
+        sleep(3)
+
+    if commands_from_local:
         order = 0
-        for name, file in commands:
-            print(f"{name}\t\t{file}")
+        for name, file in commands_from_local:
             with open("doc/" + file) as f:
                 body = f.read()
-                publishDoc(name, body, order)
+                publishDoc(Action.ADD if name in commands_to_add else Action.UPDATE, name, body, order, headers)
             order = order + 1
             sleep(3)
     else:

.github/workflows/bsd.yml

@@ -5,28 +5,35 @@ on:
     branches:
       - "master"
   pull_request:
+  workflow_dispatch:
 
 jobs:
   testfreebsd:
-    runs-on: macos-10.15
+    runs-on: ubuntu-22.04
     name: Build and test on FreeBSD
     timeout-minutes: 120
+    strategy:
+      fail-fast: true
+      matrix:
+        bitcoind-version: ["27.1"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Test in FreeBSD
       id: test
-      uses: vmactions/freebsd-vm@v0.1.5
+      uses: vmactions/freebsd-vm@v1
       with:
         usesh: true
+        sync: rsync
+        copyback: false
         prepare: |
           pkg install -y \
+            bash \
             wget \
-            py38-pip \
-            py38-sqlite3 \
+            python310 \
             gmake \
             git \
             python \
-            postgresql12-server \
+            postgresql16-server \
             autoconf \
             automake \
             libtool \
@@ -34,38 +41,26 @@ jobs:
             gettext \
             sqlite3 \
             lowdown \
+            pkgconf \
+            jq \
+            protobuf \
             curl
 
-          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2021-08-3z1
+            python3.10 -m ensurepip
+            python3.10 -m pip install --upgrade pip
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+
 
-          cd /tmp/ || exit 1
-          wget https://bitcoincore.org/bin/bitcoin-core-26.1/bitcoin-26.1-x86_64-linux-gnu.tar.gz
-          tar -xf bitcoin-26.1-x86_64-linux-gnu.tar.bz2
-          sudo mv bitcoin-26.1/bin/* /usr/local/bin
-          rm -rf \
-            bitcoin-26.1-x86_64-linux-gnu.tar.gz \
-            bitcoin-26.1
-          
         run: |
-          PATH=/root/.local/bin:$PATH
+          PATH=/root/.local/bin:$PATH:/root/.cargo/bin; export PATH
           pip install --user -U wheel pip
-          pip install --user -U -r requirements.txt
-
-          # Install utilities that aren't dependencies, but make
-          # running tests easier/feasible on CI (and pytest which
-          # keeps breaking the rerunfailures plugin).
-          pip install --user -U \
-            blinker \
-            flake8 \
-            mako \
-            pytest-sentry \
-            pytest-test-groups==1.0.3 \
-            pytest-custom-exit-code==0.3.0 \
-            pytest-json-report
+          pip3 install --user poetry
+          poetry install
 
           git clone https://github.com/lightning/bolts.git ../bolts
           # fatal: unsafe repository ('/Users/runner/work/lightning/lightning' is owned by someone else)
           git config --global --add safe.directory `pwd`
+          for d in libsodium libwally-core gheap jsmn libbacktrace lowdown; do git config --global --add safe.directory `pwd`/external/$d; done
           git submodule update --init --recursive
 
           ./configure CC="$CC" --disable-valgrind
@@ -80,8 +75,5 @@ jobs:
           EOF
 
           # Just run a "quick" test without memory checking
-          gmake
-
-          # Clean up to maximize rsync's chances of succeeding
-          gmake clean
+          poetry run gmake
 

.github/workflows/ci.yaml

@@ -20,7 +20,7 @@ env:
 jobs:
   prebuild:
     name: Pre-build checks
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 30
     env:
       BOLTDIR: bolts
@@ -28,7 +28,7 @@ jobs:
       fail-fast: true
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
@@ -42,15 +42,42 @@ jobs:
           git fetch origin ${{ github.base_ref }}
           git rebase origin/${{ github.base_ref }}
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - name: Check changelog
+        env:
+          PR_DESCRIPTION: "${{ github.event.pull_request.body || '' }}"
+          EVENT_NAME: "${{ github.event_name }}"
+          BASE_REF: "${{ github.base_ref || 'master' }}"
+        run: |
+          echo "Event Name: $EVENT_NAME"
+          echo "Base Ref: $BASE_REF"
+          echo "PR DESCRIPTION: $PR_DESCRIPTION"
+          if [ "$EVENT_NAME" = "pull_request" ]; then
+            if [[ "$PR_DESCRIPTION" != *"Changelog-None"* && \
+                  -z "$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')" && \
+                  "$(git rev-parse --abbrev-ref HEAD)" != "$BASE_REF" ]]; then
+              echo "::error::'Changelog' entry is missing in all commits, and 'Changelog-None' not specified in the PR description"
+              exit 1
+            else
+              if [[ "$PR_DESCRIPTION" == *"Changelog-None"* ]]; then
+                echo "Changelog found in PR description"
+              else
+                echo "Changelog found in Commit \"$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')\""
+              fi
+            fi
+          else
+            echo "Not a PR event."
+          fi
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install dependencies
         run: |
           bash -x .github/scripts/setup.sh
           pip install -U pip wheel poetry
+          poetry self add poetry-plugin-export
           # Export and then use pip to install into the current env
           poetry export -o /tmp/requirements.txt --without-hashes --with dev
           pip install -r /tmp/requirements.txt
@@ -59,6 +86,9 @@ jobs:
       - name: Configure
         run: ./configure --enable-debugbuild --enable-rust
       - name: Check source
+        env:
+          VALGRIND: 0
+          PYTEST_OPTS: --timeout=1200
         run: make check-source BASE_REF="origin/${{ github.base_ref }}"
       - name: Check Generated Files have been updated
         run: make check-gen-updated
@@ -93,12 +123,12 @@ jobs:
             VALGRIND: 0
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install dependencies
         run: |
@@ -115,6 +145,7 @@ jobs:
         run: |
           set -e
           pip3 install --user pip wheel poetry
+          poetry self add poetry-plugin-export
           poetry export -o requirements.txt --with dev --without-hashes
           python3 -m pip install -r requirements.txt
           ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }}
@@ -125,7 +156,7 @@ jobs:
           mv testpack.tar.bz2 cln-${CFG}.tar.bz2
       - name: Check rust packages
         run: cargo test --all
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: cln-${{ matrix.CFG }}.tar.bz2
           path: cln-${{ matrix.CFG }}.tar.bz2
@@ -150,12 +181,12 @@ jobs:
             VALGRIND: 0
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install dependencies
         run: |
@@ -163,6 +194,7 @@ jobs:
           sudo apt-get update -qq
           sudo apt-get install -y -qq lowdown
           pip install -U pip wheel poetry
+          poetry self add poetry-plugin-export
           # Export and then use pip to install into the current env
           poetry export -o /tmp/requirements.txt --without-hashes --with dev
           pip install -r /tmp/requirements.txt
@@ -170,14 +202,14 @@ jobs:
           git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
 
       - name: Download build
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: cln-${{ matrix.CFG }}.tar.bz2
 
       - name: Check
         run: |
           tar -xaf cln-${{ matrix.CFG }}.tar.bz2
-          make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }}
+          eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }}
 
   check-fuzz:
     name: Run fuzz regression tests
@@ -186,17 +218,18 @@ jobs:
       - prebuild
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install dependencies
         run: |
           bash -x .github/scripts/setup.sh
           pip install -U pip wheel poetry
+          poetry self add poetry-plugin-export
           # Export and then use pip to install into the current env
           poetry export -o /tmp/requirements.txt --without-hashes --with dev
           pip install -r /tmp/requirements.txt
@@ -211,14 +244,12 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 120
     env:
-      BITCOIN_VERSION: "26.1"
-      ELEMENTS_VERSION: 22.0.2
       RUST_PROFILE: release  # Has to match the one in the compile step
-      PYTEST_OPTS: --timeout=1200 --force-flaky
+      PYTEST_OPTS: --timeout=1200
     needs:
       - compile
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         include:
           - NAME: gcc
@@ -261,12 +292,12 @@ jobs:
             EXPERIMENTAL_SPLICING: 1
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install dependencies
         run: |
@@ -279,7 +310,7 @@ jobs:
         run: .github/scripts/install-bitcoind.sh
 
       - name: Download build
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: cln-${{ matrix.CFG }}.tar.bz2
 
@@ -312,22 +343,20 @@ jobs:
         run: |
           env
           cat config.vars
-          VALGRIND=0 poetry run pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
+          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
 
   integration-valgrind:
     name: Valgrind Test CLN ${{ matrix.name }}
     runs-on: ubuntu-22.04
     timeout-minutes: 120
     env:
-      BITCOIN_VERSION: "26.1"
-      ELEMENTS_VERSION: 22.0.2
       RUST_PROFILE: release  # Has to match the one in the compile step
       CFG: compile-gcc
-      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --force-flaky
+      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800
     needs:
       - compile
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         include:
           - NAME: Valgrind (01/10)
@@ -352,12 +381,12 @@ jobs:
             PYTEST_OPTS: --test-group=10 --test-group-count=10
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install dependencies
         run: |
@@ -370,7 +399,7 @@ jobs:
         run: .github/scripts/install-bitcoind.sh
 
       - name: Download build
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: cln-compile-gcc.tar.bz2
 
@@ -382,23 +411,21 @@ jobs:
           SLOW_MACHINE: 1
           TEST_DEBUG: 1
         run: |
-          VALGRIND=1 poetry run pytest tests/ -vvv -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
+          VALGRIND=1 poetry run eatmydata pytest tests/ -vvv -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
 
   integration-sanitizers:
     name: Sanitizers Test CLN
     runs-on: ubuntu-22.04
     timeout-minutes: 120
     env:
-      BITCOIN_VERSION: "26.1"
-      ELEMENTS_VERSION: 22.0.2
       RUST_PROFILE: release
       SLOW_MACHINE: 1
       TEST_DEBUG: 1
-      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --force-flaky
+      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800
     needs:
       - compile
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         include:
           - NAME: ASan/UBSan (01/10)
@@ -423,27 +450,25 @@ jobs:
             PYTEST_OPTS: --test-group=10 --test-group-count=10
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install dependencies
         run: |
           bash -x .github/scripts/setup.sh
           set -e
-          pip3 install --user pip wheel poetry
-          poetry export -o requirements.txt --with dev --without-hashes
-          python3 -m pip install -r requirements.txt
-          poetry install
+          pip3 install --user wheel poetry
+          poetry install --with dev --no-root
 
       - name: Install bitcoind
         run: .github/scripts/install-bitcoind.sh
 
       - name: Download build
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: cln-compile-clang-sanitizers.tar.bz2
 
@@ -452,20 +477,177 @@ jobs:
 
       - name: Test
         run: |
-          poetry run pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
+          poetry run eatmydata pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
+
+  update-docs-examples:
+    name: Update examples in doc schemas (disabled temporarily!)
+    if: false
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+    env:
+      VALGRIND: 0
+      GENERATE_EXAMPLES: 1
+      PYTEST_OPTS: --timeout=1200
+      TEST_NETWORK: regtest
+    needs:
+      - compile
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install dependencies
+        run: |
+          bash -x .github/scripts/setup.sh
+          pip install -U pip wheel poetry
+          poetry self add poetry-plugin-export
+          poetry export -o /tmp/requirements.txt --without-hashes --with dev
+          pip install -r /tmp/requirements.txt
+      - name: Install bitcoind
+        env:
+          TEST_NETWORK: regtest
+        run: .github/scripts/install-bitcoind.sh
+      - name: Download build
+        uses: actions/download-artifact@v4
+        with:
+          name: cln-compile-gcc.tar.bz2
+      - name: Unpack pre-built CLN
+        run: |
+          tar -xaf cln-compile-gcc.tar.bz2
+      - name: Test
+        run: |
+          eatmydata make -j $(nproc) check-doc-examples
+
+  min-btc-support:
+    name: Test minimum supported BTC v${{ matrix.MIN_BTC_VERSION }} with ${{ matrix.NAME }}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 120
+    env:
+      RUST_PROFILE: release  # Has to match the one in the compile step
+      PYTEST_OPTS: --timeout=1200
+    needs:
+      - compile
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - NAME: clang
+            CFG: compile-clang
+            TEST_DB_PROVIDER: sqlite3
+            COMPILER: clang
+            TEST_NETWORK: regtest
+            MIN_BTC_VERSION: '25.0'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          pip3 install --user pip wheel poetry
+          poetry install
+
+      - name: Download Bitcoin Core
+        run: wget "https://bitcoincore.org/bin/bitcoin-core-${{ matrix.MIN_BTC_VERSION }}/bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz"
+
+      - name: Extract Bitcoin Core
+        run: tar -xf "bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz"
+
+      - name: Move Bitcoin Core Binaries
+        run: sudo mv bitcoin-${{ matrix.MIN_BTC_VERSION }}/bin/* /usr/local/bin/
+
+      - name: Clean Up Downloaded Bitcoin
+        run: rm -rf "bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz" "bitcoin-${{ matrix.MIN_BTC_VERSION }}"
+
+      - name: Download build
+        uses: actions/download-artifact@v4
+        with:
+          name: cln-${{ matrix.CFG }}.tar.bz2
+
+      - name: Unpack pre-built CLN
+        env:
+          CFG: ${{ matrix.CFG }}
+        run: |
+          tar -xaf cln-${CFG}.tar.bz2
+
+      - name: Test
+        env:
+          COMPILER: ${{ matrix.COMPILER }}
+          COMPAT: 1
+          CFG: ${{ matrix.CFG }}
+          SLOW_MACHINE: 1
+          PYTEST_PAR: 10
+          TEST_DEBUG: 1
+          TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
+          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
+          LIGHTNINGD_POSTGRES_NO_VACUUM: 1
+        run: |
+          env
+          cat config.vars
+          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
+
+  check-flake:
+    name: Check Nix Flake
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - name: Check Nix flake inputs
+        uses: DeterminateSystems/flake-checker-action@v8
+      - name: Install Nix
+        uses: cachix/install-nix-action@V27
+        with:
+          nix_path: nixpkgs=channel:nixos-24.05
+      - name: Check flake
+        run: nix flake check .?submodules=1#
 
   gather:
     # A dummy task that depends on the full matrix of tests, and
     # signals successful completion. Used for the PR status to pass
-    # before merging.
+    # before merging.  Needs to run even if they failed!
     name: CI completion
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs:
       - integration
       - check-units
       - integration-valgrind
       - integration-sanitizers
+      - min-btc-support
+      - check-flake
+    if: ${{ always() }}
     steps:
       - name: Complete
+        env:
+          JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC FLAKE"
+          INTEGRATION: ${{ needs.integration.result }}
+          CHECK_UNITS: ${{ needs['check-units'].result }}
+          VALGRIND: ${{ needs['integration-valgrind'].result }}
+          SANITIZERS: ${{ needs['integration-sanitizers'].result }}
+          DOCS: ${{ needs['update-docs-examples'].result }}
+          BTC: ${{ needs['min-btc-support'].result }}
+          FLAKE: ${{ needs['check-flake'].result }}
         run: |
-          echo CI completed successfully
+          failed=""
+          for name in $JOB_NAMES; do
+            result="${!name}"
+            echo "$name: $result"
+            if [[ "$result" != "success" ]]; then
+              failed="yes"
+            fi
+          done
+          if [[ "$failed" == "yes" ]]; then
+            echo "One or more required jobs failed"
+            exit 1
+          fi

.github/workflows/crate-io.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   release_rust:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable

.github/workflows/docker-release.yml

@@ -0,0 +1,118 @@
+name: Build and push multi-platform docker images
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+[0-9a-z]+'
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Release version'
+        required: true
+
+      repository-name:
+        description: 'Docker repository name'
+        default: 'elementsproject'
+        required: false
+
+      platforms-to-build:
+        description: 'List of platforms to build'
+        default: 'linux/amd64,linux/arm64,linux/arm/v7'
+        required: false
+  
+      push-latest:
+        description: 'Push the latest tag also?'
+        default: 'false'
+        required: false
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.ref }}  # Ensures the branch triggering the workflow is checked out
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Setup Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+
+    - name: Set up values
+      id: set-values
+      run: |
+        if [[ "${{ github.event.inputs.version }}" != "" ]]; then
+          echo "Input version provided"
+          VERSION=${{ github.event.inputs.version }}
+        elif [[ ${{ github.ref_type }} == "tag" ]]; then
+          echo "This is a tag event"
+          VERSION=${{ github.ref_name }}
+        else
+          echo "No version provided and no tag found."
+          exit 1
+        fi
+        echo "VERSION=$VERSION" >> $GITHUB_ENV
+
+        if [[ "${{ github.event.inputs.repository-name }}" != "" ]]; then
+          REPONAME=${{ github.event.inputs.repository-name }}
+        else
+          REPONAME="elementsproject"
+        fi
+        echo "REPONAME=$REPONAME" >> $GITHUB_ENV
+
+        if [[ "${{ github.event.inputs.platforms-to-build }}" != "" ]]; then
+          PLATFORMS=${{ github.event.inputs.platforms-to-build }}
+        else
+          PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
+        fi
+        echo "PLATFORMS=$PLATFORMS" >> $GITHUB_ENV
+
+        if [[ "${{ github.event.inputs.push-latest }}" == "true" ]] || 
+          ([[ "${{ github.ref_type }}" == "tag" ]] && [[ ! "$VERSION" =~ rc ]]); then
+          echo "Latest true"
+          PUSHLATEST="true"
+        else
+          echo "Latest false"
+          PUSHLATEST="false"
+        fi
+        echo "PUSHLATEST=$PUSHLATEST" >> $GITHUB_ENV
+
+        TAGS="$REPONAME/lightningd:$VERSION"
+        if [[ "$PUSHLATEST" == "true" ]]; then
+          TAGS="$TAGS,$REPONAME/lightningd:latest"
+        fi
+        echo "TAGS=$TAGS" >> $GITHUB_ENV
+
+    - name: Print GitHub Ref Values
+      run: |
+        echo "GITHUB REF TYPE: ${{ github.ref_type }}"
+        echo "GITHUB REF NAME: ${{ github.ref_name }}"
+        echo "EVENT INPUT VERSION: ${{ github.event.inputs.version }}"
+        echo "EVENT INPUT REPO: ${{ github.event.inputs.repository-name }}"
+        echo "EVENT INPUT PLATFORMS: ${{ github.event.inputs.platforms-to-build }}"
+        echo "EVENT INPUT PUSH LATEST: ${{ github.event.inputs.push-latest }}"
+        echo "ENV VERSION: ${{ env.VERSION }}"
+        echo "ENV REPO NAME: ${{ env.REPONAME }}"
+        echo "ENV PLATFORMS: ${{ env.PLATFORMS }}"
+        echo "ENV PUSH LATEST: ${{ env.PUSHLATEST }}"
+        echo "ENV TAGS: ${{ env.TAGS }}"
+
+    - name: Build and push Docker image
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./Dockerfile
+        push: true
+        platforms: ${{ env.PLATFORMS }}
+        tags: ${{ env.TAGS }}

.github/workflows/docker.yml

@@ -0,0 +1,34 @@
+name: Docker packaging
+on:
+  push:
+    tags:
+      - 'basedon-*'
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Environment variables
+        run: env
+      -
+        name: Create images
+        env:
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }}
+          DOCKERHUB_REPO: ${{ vars.DOCKERHUB_REPO }}
+        shell: bash
+        run: |
+          LATEST_TAG=${GITHUB_REF#refs/tags/}
+          LATEST_TAG=${LATEST_TAG:8} #trim "basedon-" from tag
+          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USER" --password-stdin
+          docker buildx create --use
+          DOCKER_BUILDX_OPTS="--platform linux/amd64,linux/arm64,linux/arm/v7 --push"
+          docker buildx build $DOCKER_BUILDX_OPTS -t $DOCKERHUB_REPO:$LATEST_TAG .

.github/workflows/macos.yaml

@@ -5,55 +5,58 @@ on:
 jobs:
   smoke-test:
     name: Smoke Test macOS
-    runs-on: macos-latest
+    runs-on: macos-14
     timeout-minutes: 120
     strategy:
       fail-fast: true
+      matrix:
+        bitcoind-version: ["27.1"]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2.0.0
+        uses: actions/checkout@v4
+
+      - name: Download Bitcoin ${{ matrix.bitcoind-version }} & install binaries
+        run: |
+          export BITCOIND_VERSION=${{ matrix.bitcoind-version }}
+          export TARGET_ARCH="arm64-apple-darwin"
+
+          wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
+          tar -xzf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
+          sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin
+          rm -rf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz bitcoin-${BITCOIND_VERSION}
 
       - name: Install dependencies
         run: |
-          export PATH="/usr/local/opt:/Users/runner/.local/bin:/Users/runner/Library/Python/3.10/bin:$PATH"
+          export PATH="/usr/local/opt:/Users/runner/.local/bin:/opt/homebrew/bin/python3.10/bin:$PATH"
 
-          export BITCOIN_VERSION=26.1
-          brew install wget autoconf automake libtool python@3.10 gnu-sed gettext libsodium
-
-          (
-            cd /tmp/
-            wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-osx64.tar.gz -O bitcoin.tar.gz
-            tar -xvzf bitcoin.tar.gz
-            sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin
-          )
-
-          python3.10 -m pip install -U --user poetry wheel pip
+          brew install gnu-sed python@3.10 autoconf automake libtool protobuf
+          python3.10 -m pip install -U --user poetry==1.8.0 wheel pip mako
           python3.10 -m poetry install
-          python3.10 -m pip install -U --user mako
-          
-          ln -s /usr/local/Cellar/gettext/0.20.1/bin/xgettext /usr/local/opt
 
-      - name: Build
-        env:
-          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
-          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
-          NO_PYTHON: ${{ matrix.NO_PYTHON }}
-          COPTFLAGS: ${{ matrix.COPTFLAGS }}
-          NETWORK: ${{ matrix.NETWORK }}
-          TEST_CMD: ${{ matrix.TEST_CMD }}
-          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
-          TEST_GROUP: ${{ matrix.TEST_GROUP }}
+      - name: Build and install CLN
         run: |
-          export PATH="/usr/local/opt:/Users/runner/.local/bin:/Users/runner/Library/Python/3.10/bin:/usr/local/opt:$PATH"
-          export LDFLAGS="-L/usr/local/opt/sqlite/lib"
-          export CPPFLAGS="-I/usr/local/opt/sqlite/include"
-
-          cat << EOF > pytest.ini
-          [pytest]
-          addopts=-p no:logging --color=yes --timeout=600 --timeout-method=thread --test-group-random-seed=42 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
-          markers =
-              slow_test: marks tests as slow (deselect with '-m "not slow_test"')
-          EOF
+          export CPATH=/opt/homebrew/include
+          export LIBRARY_PATH=/opt/homebrew/lib
 
           python3.10 -m poetry run ./configure --disable-valgrind --disable-compat
           python3.10 -m poetry run make
+
+      - name: Start bitcoind in regtest mode
+        run: |
+          bitcoind -regtest -daemon
+          sleep 5
+
+      - name: Generate initial block
+        run: |
+          bitcoin-cli -regtest createwallet default_wallet
+          bitcoin-cli -regtest generatetoaddress 1 $(bitcoin-cli -regtest getnewaddress)
+          sleep 2
+
+      - name: Start CLN in regtest mode
+        run: |
+          lightningd/lightningd --network=regtest --log-file=/tmp/l1.log --daemon
+          sleep 5
+
+      - name: Verify CLN is running
+        run: |
+          cli/lightning-cli --regtest getinfo

.github/workflows/pypi.yml

@@ -1,20 +1,22 @@
----
 name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+
 on:
-  # Only deploy if we're the result of a PR being merged
   push:
-    workflow_dispatch:
-    branches:
-      - master
     tags:
-      # Semantic versioning tags
+      - 'v[0-9]+.[0-9]+'
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      # Date style tags
-      - 'v[0-9]{2}.[0-9]{2}'
+      - 'v[0-9]+.[0-9]+[0-9a-z]+'
+  workflow_dispatch:
+    inputs:
+      dist-location:
+        description: 'Distribution location (test/prod)'
+        default: 'test'
+        required: false
+
 jobs:
   deploy:
     name: Build and publish ${{ matrix.package }} 🐍
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 120
     strategy:
       fail-fast: true
@@ -26,65 +28,73 @@ jobs:
             WORKDIR: contrib/pyln-testing
           - PACKAGE: pyln-proto
             WORKDIR: contrib/pyln-proto
-          - PACKAGE: pyln-grpc-proto
-            WORKDIR: contrib/pyln-grpc-proto
-          # Bolt packages are handled differently
-          #- PACKAGE: pyn-bolt1
-          #  WORKDIR: contrib/pyln-spec/bolt1/
-          #- PACKAGE: pyn-bolt2
-          #  WORKDIR: contrib/pyln-spec/bolt2/
-          #- PACKAGE: pyn-bolt4
-          #  WORKDIR: contrib/pyln-spec/bolt4/
-          #- PACKAGE: pyn-bolt7
-          #  WORKDIR: contrib/pyln-spec/bolt7/
     steps:
-    - uses: actions/checkout@master
+    - name: Checkout repository
+      uses: actions/checkout@v4
       with:
         # Need to fetch entire history in order to locate the version tag
         fetch-depth: 0
 
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.8
-
-    - name: Install pypa/build and poetry
-      run: >-
-        python -m pip install build poetry --user
-
     - name: Check version tag
       run: >-
-        git describe --always --dirty=-modded --abbrev=7
+        git describe --tags --always --dirty=-modded --abbrev=7
 
-    - name: Build a binary wheel and a source tarball
+    - name: Setup Version
       env:
         WORKDIR: ${{ matrix.WORKDIR }}
       run: |
-        export VERSION=$(git describe --abbrev=0).post$(git describe --abbrev=1 | awk -F "-" '{print $2}')
-        cd ${{ env.WORKDIR }}
-        make upgrade-version NEW_VERSION=$VERSION
-        poetry build --no-interaction
+        echo "VERSION=$(git describe --tags --abbrev=0).post$(git describe --tags --abbrev=1 | awk -F "-" '{print $2}')" >> $GITHUB_ENV
+
+    - name: Set up values
+      id: set-values
+      run: |
+        if [[ "${{ github.event.inputs.dist-location }}" != "" ]]; then
+          DISTLOCATION=${{ github.event.inputs.dist-location }}
+        elif [[ "${{ github.ref_type }}" == "tag" ]] && [[ ! "${{ github.ref_name }}" =~ rc ]]; then
+          DISTLOCATION="prod"
+        else
+          DISTLOCATION="test"
+        fi
+        echo "DISTLOCATION=$DISTLOCATION" >> $GITHUB_OUTPUT
+        echo "EVENT DISTLOCATION: ${{ github.event.inputs.dist-location }}"
+        echo "DISTRIBUTION LOCATION: $DISTLOCATION"
+  
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: Install Poetry
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
+        echo "PATH=$HOME/.local/bin:$PATH"
 
     - name: Publish distribution 📦 to Test PyPI
-      if: github.repository == 'ElementsProject/lightning' && github.ref == 'refs/heads/master'
+      if: github.repository == 'ElementsProject/lightning' && steps.set-values.outputs.DISTLOCATION == 'test'
       env:
         POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.TEST_PYPI_API_TOKEN }}
         WORKDIR: ${{ matrix.WORKDIR }}
       run: |
+        echo "POETRY VERSION TEST: $(poetry --version)"
+        echo "Pyln VERSION: $VERSION"
         cd ${{ env.WORKDIR }}
         python3 -m pip config set global.timeout 150
         poetry config repositories.testpypi https://test.pypi.org/legacy/
+        make upgrade-version NEW_VERSION=$VERSION
         poetry build  --no-interaction
         poetry publish --repository testpypi --no-interaction --skip-existing
 
     - name: Publish distribution 📦 to PyPI
-      if: startsWith(github.ref, 'refs/tags/v') && github.repository == 'ElementsProject/lightning'
+      if: github.repository == 'ElementsProject/lightning' && steps.set-values.outputs.DISTLOCATION == 'prod'
       env:
         POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
         WORKDIR: ${{ matrix.WORKDIR }}
       run: |
+        echo "POETRY VERSION PUBLISH: $(poetry --version)"
         cd ${{ env.WORKDIR }}
-        export VERSION=$(git describe --abbrev=0)
+        export VERSION=$(git describe --tags --abbrev=0)
+        echo "Pyln VERSION: $VERSION"
         make upgrade-version NEW_VERSION=$VERSION
         python3 -m pip config set global.timeout 150
         poetry build  --no-interaction

.github/workflows/rdme-docs-sync.yml

@@ -6,13 +6,14 @@ on:
       - 'master'
     paths:
       - 'doc/**'
+  workflow_dispatch:
 
 jobs:
   rdme-docs-sync:  
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Check out repo 📚
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Sync doc/getting-started/ 🚀
         uses: readmeio/rdme@v8

.github/workflows/readme-rpc-sync.yml

@@ -5,24 +5,39 @@ on:
     branches:
       - 'master'
     paths:
-      - 'doc/**.md'
+      - 'doc/schemas/*.json'
+      - 'doc/*.1.md'
+      - 'doc/*.5.md'
+      - 'doc/*.8.md'
+      - 'doc/lightningd-*.7.md'
+      - 'doc/reckless.7.md'
+  workflow_dispatch:
 
 jobs:
   rdme-rpc-sync:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: 3.9
 
-    - name: Install requests module
-      run: python -m pip install requests
+    - name: Install python modules
+      run: |
+        python -m pip install requests mako grpcio-tools
+
+    - name: Install dependencies
+      run: bash -x .github/scripts/setup.sh
+
+    - name: Build (including rpc .md files)
+      run: |
+        ./configure --enable-debugbuild
+        make -j $(nproc)
 
     - name: Set environment variable and run
       env:
-        README_API_KEY: ${{ secrets.README_API_KEY }}  
+        README_API_KEY: ${{ secrets.README_API_KEY }}
       run: python .github/scripts/sync-rpc-cmds.py

.github/workflows/release.yml

@@ -0,0 +1,191 @@
+---
+# https://docs.corelightning.org/docs/release-checklist
+name: "Release 🚀"
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+[0-9a-z]+'
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Release version'
+        required: true
+      create_release:
+        description: Create a draft release
+        default: no
+        type: choice
+        options:
+          - yes
+          - no
+
+jobs:
+  check:
+    name: Check
+    outputs:
+      version: ${{ steps.capture.outputs.version }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Determine version
+        run: |
+          if [[ "${{ github.event.inputs.version }}" != "" ]]; then
+            VERSION="${{ github.event.inputs.version }}"
+          elif [ "${{ github.ref_type }}" == "tag" ]; then
+            VERSION="${{ github.ref_name }}"
+          else
+            echo "No release version provided and no tag found."
+            exit 1
+          fi
+          echo "VERSION=$VERSION" >> "$GITHUB_ENV"
+          echo "Determined version: $VERSION"
+
+      - name: Validate release
+        run: tools/check-release.sh --version=${VERSION}
+
+      - name: Catpure version output
+        id: capture
+        run: echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+
+  releases:
+    name: Releases
+    needs: check
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false    # Let each build finish.
+      matrix:
+        target:
+          - 'bin-Fedora'
+          - 'bin-Ubuntu-focal'
+          - 'bin-Ubuntu-jammy'
+          - 'bin-Ubuntu-noble'
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+
+      # tools/build-release.sh requires lowdown
+      - name: Prepare base environment
+        run: |
+          sudo apt-get install -y lowdown
+          ./configure
+
+      - name: Build environment setup
+        run: |
+          distribution=$(echo ${{ matrix.target }} | cut -d'-' -f3)
+          echo "Building base image for ${distribution}"
+          sudo docker run --rm -v $(pwd):/build ubuntu:${distribution} bash -c "\
+              apt-get update && \
+              apt-get install -y debootstrap && \
+              debootstrap ${distribution} /build/${distribution}"
+          sudo tar -C ${distribution} -c . | docker import - ${distribution}
+
+          # Build Docker image
+          docker build -t cl-repro-${distribution} - < contrib/reprobuild/Dockerfile.${distribution}
+        if: contains(matrix.target, 'Ubuntu')
+
+      - name: Build release
+        run: tools/build-release.sh ${{ matrix.target }}
+
+      - name: Upload target artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          path: release/
+          name: ${{ matrix.target }}
+          if-no-files-found: error
+
+  artifact:
+    name: Construct release artifact
+    needs:
+      - check
+      - releases
+    env:
+      version: ${{ needs.check.outputs.version }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Merge artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: c-lightning-${{ env.version }}
+          pattern: bin-*
+          delete-merged: true
+
+  release:
+    name: Sign and prepare release draft
+    needs:
+      - check
+      - artifact
+    env:
+      version: ${{ needs.check.outputs.version }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: c-lightning-${{ env.version }}
+          path: release/
+
+      - name: Import GPG keys
+        id: gpg
+        uses: crazy-max/ghaction-import-gpg@v6
+        with:
+          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.GPG_PASSPHRASE }}
+          trust_level: 5
+
+      - name: Set default GPG key
+        run: echo "default-key ${{ steps.gpg.outputs.keyid }}" >> ~/.gnupg/gpg.conf
+
+      - name: Sign release
+        run: |
+          sudo apt-get install -y lowdown
+          ./configure
+          tools/build-release.sh --without-zip sign
+          mv release/SHA256SUMS.asc${{ steps.gpg.outputs.keyid  }} release/SHA256SUMS.asc
+
+      - name: Upload signed artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: c-lightning-${{ env.version }}
+          overwrite: true
+          path: release/
+
+      - name: Determine release data
+        id: release_data
+        run: |
+          VERSION=${{ env.version }}
+          CHANGELOG_VERSION=${VERSION#v}
+          echo "CHANGELOG_VERSION=$CHANGELOG_VERSION"
+          echo "changelog_version=$CHANGELOG_VERSION" >> "$GITHUB_OUTPUT"
+
+          CHANGELOG_TITLE=$(grep "## \[${CHANGELOG_VERSION}\]" CHANGELOG.md)
+          echo "CHANGELOG_TITLE=$CHANGELOG_TITLE"
+          echo "changelog_title=$CHANGELOG_TITLE" >> "$GITHUB_OUTPUT"
+
+          RELEASE_TITLE=$(echo $CHANGELOG_TITLE | cut -d'"' -f2)
+          echo "RELEASE_TITLE=$RELEASE_TITLE"
+          echo "release_title=$RELEASE_TITLE" >> "$GITHUB_OUTPUT"
+
+      - name: Prepare release draft
+        if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.create_release == 'yes')
+        uses: softprops/action-gh-release@v2
+        with:
+          name: "${{ env.version }} ${{ steps.release_data.outputs.release_title }}"
+          tag_name: ${{ env.version }}
+          draft: true
+          prerelease: contains(env.version, "-rc")
+          files: release/*
+          fail_on_unmatched_files: true

.github/workflows/repro.yml

@@ -0,0 +1,113 @@
+---
+# https://docs.corelightning.org/docs/repro
+name: Repro Build Nightly
+on:
+  # 05:00 Berlin, 03:00 UTC, 23:00 New York, 20:00 Los Angeles
+  schedule:
+    - cron: "0 3 * * *"
+  workflow_dispatch:
+
+jobs:
+  ubuntu:
+    name: "Ubuntu repro build: ${{ matrix.version }}"
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false    # Let each build finish.
+      matrix:
+        version: ['focal', 'jammy', 'noble']
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v4
+
+      - name: Build environment setup - ${{ matrix.version }}
+        run: |
+          echo "Building base image for ${{ matrix.version }}"
+          echo "STEP=Build environment setup" >> "$GITHUB_ENV"
+          sudo docker run --rm -v $(pwd):/build ubuntu:${{ matrix.version }} bash -c "\
+            apt-get update && \
+            apt-get install -y debootstrap && \
+            debootstrap ${{ matrix.version }} /build/${{ matrix.version }}"
+          sudo tar -C ${{ matrix.version }} -c . | docker import - ${{ matrix.version }}
+
+      - name: Builder image setup - ${{ matrix.version }}
+        run: |
+          echo "STEP=Builder image setup" >> "$GITHUB_ENV"
+          docker build -t cl-repro-${{ matrix.version }} - < contrib/reprobuild/Dockerfile.${{ matrix.version }}
+
+      - name: Build reproducible image and store Git state - ${{ matrix.version }}
+        run: |
+          echo "STEP=Build reproducible image and store Git state" >> "$GITHUB_ENV"
+
+          # Create release directory.
+          mkdir $GITHUB_WORKSPACE/release
+
+          # Perform the repro build.
+          docker run --name cl-build -v $GITHUB_WORKSPACE:/repo -e FORCE_MTIME=$(date +%F) -t cl-repro-${{ matrix.version }}
+
+          # Commit the image in order to inspect the build later.
+          docker commit cl-build cl-repro
+
+          # Inspect the version.
+          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "make version > /repo/release/version.txt"
+
+          # Inspect the Git tree state.
+          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "\
+              git --no-pager status > /repo/release/git.log && \
+              git --no-pager diff >> /repo/release/git.log"
+
+          # Change permissions on the release files for access by the runner environment.
+          sudo chown -R runner $GITHUB_WORKSPACE/release
+
+      - name: Assert clean version - ${{ matrix.version }}
+        run: |
+          echo "STEP=Assert clean version" >> "$GITHUB_ENV"
+          echo 'Version:'
+          cat release/version.txt
+          echo -e
+
+          releasefile=$(ls release/clightning-*)
+          echo 'Release file:'
+          ls -al release/clightning-*
+          echo -e
+          if [ -n "$(cat release/version.txt | sed -n '/-modded/p')" ] || \
+             [ -n "$(echo $releasefile | sed -n '/-modded/p')" ]; then
+            echo "Git Status and Diff:"
+            cat release/git.log
+            echo -e
+            echo 'Error: release modded / dirty tree.'
+            exit 1
+          else
+            echo 'Success! Clean release.'
+          fi
+
+      - name: Upload release artifact - ${{ matrix.version }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-${{ matrix.version }}
+          path: release
+          retention-days: 3 # Automatically delete after 3 days
+
+      - name: Send email on failure
+        if: ${{ failure() }}
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 587
+          username: ${{ secrets.EMAIL_USERNAME }}
+          password: ${{ secrets.EMAIL_PASSWORD }}
+          from: ${{ secrets.EMAIL_USERNAME }}
+          to: ${{ vars.DISTRIBUTION_LIST }}
+          subject: "CI Failure: Step ${{ env.STEP }} failed for distro ${{ matrix.version }}"
+          convert_markdown: true
+          html_body: |
+            <html>
+              <body>
+                <p>GitHub Workflow ${{ github.workflow }} Failed! For more details, click on the action below.</p>
+                <strong>Failure Details:</strong><br/>
+                <strong>Event: </strong>${{ github.event_name }}<br/>
+                <strong>Job: </strong>${{ github.job }}<br/>
+                <strong>Distro: </strong>${{ matrix.version }}<br/>
+                <strong>Step: </strong>${{ env.STEP }}<br/>
+                <strong>Action: </strong><a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}</a><br/>
+              </body>
+            </html>

.gitignore

@@ -57,9 +57,9 @@ tests/plugins/test_selfdisable_after_getmanifest
 
 # Ignore generated files
 devtools/features
-doc/schemas/lightning-sql.json
-doc/lightning-*.7.md
-doc/lightning*.[1578]
+doc/schemas/sql.json
+doc/*.7.md
+doc/*.[1578]
 doc/reckless*.[1578]
 *_sqlgen.[ch]
 *_wiregen.[ch]
@@ -69,6 +69,9 @@ tests/node_pb2.py
 tests/node_pb2_grpc.py
 tests/primitives_pb2.py
 tests/primitives_pb2_grpc.py
+tests/autogenerate-examples-status.log
+tests/autogenerate-examples.json
+tests/autogenerate-examples-repeat.log
 
 # Ignore unrelated stuff
 .DS_Store
@@ -79,14 +82,23 @@ compile_commands.json
 # Rust targets
 target
 plugins/cln-grpc
+plugins/clnrest
 
 # Build directories
 bionic/
 focal/
 jammy/
+noble/
 release/
 .vscode/
 .cache/
 
 # Ignore release verification Sha256Sums
 SHA256SUMS-*
+doc/.doc_version
+autogenerate-examples-status.log
+
+# Ignore nix outputs
+result
+result-[0-9]*
+tests/plugins/channeld_fakenet

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.8.0
+  hooks:
+    # Run the linter.
+    - id: ruff
+      args: [ --diff ]
+      exclude: "contrib/pyln-grpc-proto/pyln/grpc/(primitives|node)_pb2(|_grpc).py"
+    # Run the formatter.
+    - id: ruff-format
+      args: [ --diff ]
+      exclude: "contrib/pyln-grpc-proto/pyln/grpc/(primitives|node)_pb2(|_grpc).py"

.version

@@ -0,0 +1 @@
+25.05

CHANGELOG.md

@@ -1,8 +1,839 @@
 # Changelog
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [25.05] - 2025-06-16: "Satoshi's OP_RETURN Opinion"
+
+This release named by Peter Neuroth (@nepet).
+
+**WARNING**: `--experimental-splicing` is incompatible with previous CLN versions!  You will not
+              be able to reestablish channels with older nodes at all, if this is enabled!
+
+### Added
+
+ - Protocol: we now offer peer storage to any peers who create a channel. ([#8140])
+ - lsps-plugin: Added lsps0 service support. ([#8227])
+ - lsps-plugin: Added lsps0 client support. ([#8227])
+ - JSON-RPC: `listhtlcs` supports `index`, `start` and `end` parameters for pagination support. ([#8166])
+ - JSON-RPC: `listhtlcs` has `created_index` and `updated_index` fields. ([#8166])
+ - JSON-RPC: `wait` now supports the `htlcs` (`listhtlcs`) subsystem. ([#8166])
+ - JSON-RPC: `wait` now has separate `invoices`, `forwards` and `sendpays` objects for each subsystem. ([#8166])
+ - Reckless: `reckless update` updates all reckless-installed plugins. ([#8266])
+ - Splicing: Performing a splice on a channel with a pending splice now performs a RBF on the splice transaction. ([#8021])
+ - JSON-RPC: Added a new rpc command signmessagewithkey to sign input messages with keys from our wallet. ([#8226])
+ - HSMD: Added new wire API to sign messages with bitcoin wallet keys according to BIP137. ([#8226])
+ - JSON-RPC: `askrene-bias-channel` now has a `relative` option to add, rather than replace, a channel bias. ([#8277])
+ - sendonion: a new paramter `total_amount_msat` to make MPP payments with sendpay and sendonion compatible. ([#8015])
+ - Protocol: We now reply to `channel_reestablish` even on long-closed channels. ([#8162])
+ - JSON-RPC: `listpeerchannels` now has a `short_channel_id` parameter for just listing a specific channel. ([#8237])
+ - cln-grpc: Exposed NotificationStream in the server module. ([#8220])
+
+
+### Changed
+
+ - Splicing: The splicing protocol is now compatible with Eclair, incompatible with previous CLN versions ([#8021])
+ - Protocol: We now exchange `announcement_signatures` as soon as we're ready, rather than waiting for 6 blocks (as per recent BOLT update.) ([#8136])
+ - Protocol: We won't forget still-opening channels after 2016 blocks, unless there are more than 100. ([#8162])
+ - Reckless: Accepts a source url or local directory as an argument to `reckless install`. ([#8266])
+ - Protocol: bitcoind peers are no longer asked for blocks if they don't store them. ([#8268])
+ - Protocol: DNS seeds are no longer used for peer lookup fallbacks. ([#8272])
+ - JSON-RPC: askrene channel bias can be set to accumulate via `askrene-bias-channel` by setting `relative=true`. ([#8072])
+ - Removed lightning- prefix from schemas/.json and doc/.md files. ([#8038])
+ - Update libwally to 1.4.0. ([#8158])
+ - Plugins: Log messages containing \n are now split into multiple log lines and " is no longer turned into \". ([#7013])
+
+
+### Deprecated
+
+Note: You should always set `allow-deprecated-apis=false` to test for changes.
+
+ - Config: `--experimental-peer-storage` (it's now the default). ([#8140])
+ - JSON-RPC: channel_state_changed notification field `old_state` value "unknown" (it will be omitted, instead). ([#8271])
+ - JSON-RPC: `wait` reply `details` object: use subsytem specific object instead. ([#8166])
+
+
+### Removed
+
+ - RPC `listchannels` no longer includes private local channels (deprecated v23.08, disabled by default in v24.11). ([#8118])
+ - Plugins which didn't accept string JSON RPC fields (deprecated v23.08, disabled by default in v24.11). ([#8118])
+ - Default settings for flag options in plugins which aren't "false". ([#8118])
+ - Allowing 0/1 instead of false/true for plugin options (deprecated v23.08, disabled by default in v24.11). ([#8118])
+ - Config: `announce-addr-dns` (deprecated v23.08, disabled in v24.11). ([#8118])
+ - --bind-addr and --addr on onion addresses and local sockets (deprecated v23.08, disabled by default in v24.11). ([#8118])
+ - `accept-htlc-tlv-types` (deprecated v23.08, disabled by default in v24.11). ([#8118])
+ - Connection/disconnection/block_added notification raw fields (deprecated v23.08, disabled by default in v24.11). ([#8118])
+ - `listconfigs` raw listing (deprecated v23.08, disabled by default in v24.11). ([#8118])
+
+
+### Fixed
+
+ - Protocol: anchors' fees are now much closer to the feerate targets. ([#8236])
+ - Protocol: Removed 200ms latency from sending commit/revoke messages. ([#8222])
+ - Emergency Recover: Encourages LND nodes to properly respond with a unilateral close. ([#8213])
+ - askrene: Fixed routing in high capacity channels. ([#8129])
+ - wallet: Fees are much closer to target feerate when doing txprepare/fundchannel. ([#8236])
+ - Plugins can now log events under the LOG_TRACE flag. ([#8113])
+ - JSON-RPC: `txprepare` with `all` as well as one or more non-all amount fields now produces a valid PSBT. ([#8236])
+ - Validated public keys in BOLT11 routing hints to prevent processing of malformed public keys.([#8282])
+ - Suppress logs from chanbackup ([#8254])
+ - lightningd: Trimmed overhead of tracing infrastructure. ([#8223])
+ - Enforced minimum witness program length of 2 bytes for fallback addresses to comply with BIP-141 and prevent invalid decodings.([#8219])
+
+
+### EXPERIMENTAL
+
+ - `experimental-peer-storage` now only advertizes feature 43, not 41. ([#8135])
+
+
+[#8222]: https://github.com/ElementsProject/lightning/pull/8222
+[#8223]: https://github.com/ElementsProject/lightning/pull/8223
+[#8021]: https://github.com/ElementsProject/lightning/pull/8021
+[#8072]: https://github.com/ElementsProject/lightning/pull/8072
+[#8237]: https://github.com/ElementsProject/lightning/pull/8237
+[#8118]: https://github.com/ElementsProject/lightning/pull/8118
+[#8162]: https://github.com/ElementsProject/lightning/pull/8162
+[#7013]: https://github.com/ElementsProject/lightning/pull/7013
+[#8268]: https://github.com/ElementsProject/lightning/pull/8268
+[#8266]: https://github.com/ElementsProject/lightning/pull/8266
+[#8272]: https://github.com/ElementsProject/lightning/pull/8272
+[#8158]: https://github.com/ElementsProject/lightning/pull/8158
+[#8226]: https://github.com/ElementsProject/lightning/pull/8226
+[#8277]: https://github.com/ElementsProject/lightning/pull/8277
+[#8113]: https://github.com/ElementsProject/lightning/pull/8113
+[#8254]: https://github.com/ElementsProject/lightning/pull/8254
+[#8220]: https://github.com/ElementsProject/lightning/pull/8220
+[#8213]: https://github.com/ElementsProject/lightning/pull/8213
+[#8136]: https://github.com/ElementsProject/lightning/pull/8136
+[#8236]: https://github.com/ElementsProject/lightning/pull/8236
+[#8140]: https://github.com/ElementsProject/lightning/pull/8140
+[#8271]: https://github.com/ElementsProject/lightning/pull/8271
+[#8166]: https://github.com/ElementsProject/lightning/pull/8166
+[#8038]: https://github.com/ElementsProject/lightning/pull/8038
+[#8282]: https://github.com/ElementsProject/lightning/pull/8282
+[#8129]: https://github.com/ElementsProject/lightning/pull/8129
+[#8135]: https://github.com/ElementsProject/lightning/pull/8135
+[#8227]: https://github.com/ElementsProject/lightning/pull/8227
+[#8015]: https://github.com/ElementsProject/lightning/pull/8015
+[#8219]: https://github.com/ElementsProject/lightning/pull/8219
+[25.05rc1]: https://github.com/ElementsProject/lightning/releases/tag/v25.05rc1
+
+
+## [25.02.2] - 2025-04-22: "Onion Packet Filler Accreditation III"
+
+Important fixes for bugs in v25.02.1.
+
+### Fixed
+
+ - docker: Fixed broken docker image due to sqlite version not supporting an advanced query ([#8246])
+
+[#8246]: https://github.com/ElementsProject/lightning/pull/8246
+[25.02.2]: https://github.com/ElementsProject/lightning/releases/tag/v25.02.2
+
+
+## [25.02.1] - 2025-04-03: "Onion Packet Filler Accreditation II"
+
+Important fixes for bugs in v25.02.
+
+### Fixed
+
+ - wallet: we could miss our own returned outputs on mutual closes if peer doesn't support option_shutdown_anysegwit ([#8175])
+ - wallet: rescan for missing close outputs in case above bug happened ([#8175])
+ - lightningd: incorrect spamming of log and potential crash on testnet case of duplicate HTLCs and slow closing. ([#8192])
+ - Protocol: Egregious anchor fee paid for unilateral close txs due to HTLC timeouts; it's not as urgent as our code made out! ([#8190])
+ - lightningd: occasional crash on bitcoind callback. ([#8186])
+ - autoclean/chanbackup: fixed tracepoint crash on large number of requests. ([#8188])
+ - autoclean: fixed occasional crash when tracepoints compiled in. ([#8198])
+ - `topology` crash on invoice creation if a peer had a really high feerate. ([#8187])
+ - `make` cleans up old clnrest directory prior to building and installing the new plugin. ([#8201], [#8159])
+
+[#8186]: https://github.com/ElementsProject/lightning/pull/8186
+[#8192]: https://github.com/ElementsProject/lightning/pull/8192
+[#8187]: https://github.com/ElementsProject/lightning/pull/8187
+[#8190]: https://github.com/ElementsProject/lightning/pull/8190
+[#8175]: https://github.com/ElementsProject/lightning/pull/8175
+[#8188]: https://github.com/ElementsProject/lightning/pull/8188
+[#8198]: https://github.com/ElementsProject/lightning/pull/8198
+[#8159]: https://github.com/ElementsProject/lightning/pull/8159
+[#8201]: https://github.com/ElementsProject/lightning/pull/8201
+[25.02.1]: https://github.com/ElementsProject/lightning/releases/tag/v25.02.1
+
+
+## [25.02] - 2025-03-04: "Onion Packet Filler Accreditation"
+
+This release named by @s373nZ.
+
+### Added
+
+ - Plugins: `chanbackup` turns our peers into watchtowers and enables SCB to create penalty txns. ([#7772])
+ - JSON-RPC: `fetchinvoice` BIP 353 name support (`bip353`). ([#7887])
+ - JSON-RPC: `blacklistrune` now has an option to un-blacklist (`relist`). ([#8037])
+ - JSON-RPC: `setconfig` now has a `transient` flag which means it won't rewrite your config file. ([#8095])
+ - Plugins: New nofitications `plugin_stopped` and `plugin_started`. ([#7508])
+ - xpay: `xpay-slow-mode` makes xpay wait for all parts of a payment to complete before returning success or failure. ([#8094])
+ - Plugins: `xpay` now supports a `maxdelay` parameter for better `xpay-handle-pay` compatibility. ([#7969])
+ - JSON-RPC: `listpeerchannels` new output fields `their_max_total_htlc_out_msat` and `our_max_total_htlc_out_msat` as the value of `max_htlc_value_in_flight` (as of BOLT02) set by the local and remote nodes on channel creation. ([#8084])
+ - Config: Added support for Bitcoin `testnet4`. ([#7894])
+
+
+### Changed
+
+ - clnrest: clnrest is now a rust plugin. ([#7509])
+ - JSON-RPC: `blacklistrune` no longer supports of runes over id 100,000,000. ([#8037])
+ - Splicing: User is prevented from trying to splice unsigned PSBTs, protecting against potential issues. ([#8052])
+ - Splicing: Stricter tests for interop with Eclair. ([#8031])
+ - `xpay` now populates more fields, so `listsendpays` and `listpays` show `destination` and `amount_msat` fields for xpay payments. ([#7941])
+ - `xpay` now gives the same JSON success return as documented by `pay` when `xpay-handle-pay` is set. ([#7938])
+ - JSON-RPC: With `xpay-handle-pay` set, xpay will now be used even if `pay` uses maxfeeprecent or exemptfee parameters (e.g. Zeus.) ([#7942])
+ - Release: Docker images are now based on Debian Bookworm. ([#7921])
+ - lightning-cli: Plugin descriptions in `lightning-cli help` is now more readable with interpretation of \n characters. ([#8022])
+ - Release: Fixed version number and version check in source release zip packages. ([#8010])
+ - Logging: We no longer suppress DEBUG messages from subdaemons. ([#7935])
+ - lightning-cli: `help` messages using new-lines is now printed properly, enhancing readability and consistency. ([#8022])
+ - pyln-testing: pyln-testing is now compatible with older versions of Core Lightning. ([#7173])
+
+
+### Deprecated
+
+Note: You should always set `allow-deprecated-apis=false` to test for changes.
+
+ - JSON-RPC: `listpeerchannels` value `max_total_htlc_in_msat`: use `our_max_htlc_value_in_flight_msat` instead to follow spec naming convention. ([#8084])
+ - Config: `rest-port`, `rest-protocol`, `rest-host` and `rest-certs` disabled by default (use `clnrest-*`, or `i-promise-to-fix-broken-api-user=rest-port.clnrest-prefix` etc and PLEASE REPORT if you need this!) ([#8089])
+ - Config: `max-locktime-blocks` disabled by default (use `i-promise-to-fix-broken-api-user=max-locktime-blocks` and PLEASE REPORT if you need this!) ([#8089])
+
+
+### Removed
+
+
+### Fixed
+
+ - lightningd: Startup time vastly improved for large nodes with pending closes and many bitcoin addresses. ([#8019])
+ - renepay: No longer crashes on trying a payment after xpay. ([#7979])
+ - connectd: `dev-no-reconnect-private` is respected on restart. ([#8104])
+ - cln-grpc, clnrest: Errors that cause them to stop will now log correctly. ([#8085])
+ - Plugins: xpay doesn't simply give up if our total amount is less than `htlc_minimum_msat` on some channel. ([#8057])
+ - Plugins: xpay suppresses multi-part payment if an invoice doesn't allow it (please, fix your nodes!) ([#8059])
+ - JSON-RPC: xpay now works through unannounced channels. ([#7937])
+ - onchaind: Don't die if we fail an unrelated channel with the same peer. ([#8056])
+ - gossipd: More sanity checks that we are correctly updating the `gossip_store` file. ([#8053])
+ - gossipd: Corruption in the `gossip_store` no longer causes ever-longer startup times and no gossip updates. ([#8053])
+ - gossmap: Don't crash on ZFS while reading `gossip_store`. ([#8053])
+ - lightningd: Tell plugins our bolt12 features (so our bolt12 invoices explicitly allow MPP). ([#8059])
+ - lightningd: Allow more time for plugin startup so slow nodes don't time out on startup. ([#8060])
+ - lightningd: Fixed onchaind crash when seeing unrelated txs (usually when catching up with old closes.) ([#8137])
+ - `xpay`: Corrected the CLTV values in blinded paths which sometimes caused spurious failures. ([#8121])
+ - Protocol: We now renegotiate an interrupted close, even if we don't need it, instead of sending an error. ([#8116])
+ - bcli: `getblockfrompeer` no longer requests the block from the first peer only. ([#8069])
+ - lightning-cli: Fixed "malformed response" bug. ([#7924])
+ - lightning-cli: Fixed access to man pages from the installed directory. ([#8077])
+ - Protocol: We were overzealous in pruning channels if we hadn't seen one side's gossip update yet. ([#8027])
+ - xpay-handle-pay: Handle null parameters passed in the "param" list. ([#7958])
+ - Offers: `decode` for bolt12 invoices "features" field renamed to "invoice_features" (as documentation said.) ([#8059])
+ - JSON-RPC: `xpay` will refuse to make a 0msat payment (0msat invoice, partial payment, or manually-set on amountless invoice). ([#8024])
+ - JSON-RPC: `getroutes` will refuse, not crash, if asked to find a route for 0msat. ([#8024])
+ - Blinded paths: offline peers no longer selected. ([#8138])
+ - build: Fixed linking against libsodium on macOS. ([#7908])
+ - Fixes Postgres driver availability for arm64 and arm32 Docker images. ([#7921])
+ - xpay: xpay no longer logs "Got command" at info level. ([#7933])
+ - contrib: `startup-regtest.sh` now only inspects the most recent run's logs for the active status of the clnrest plugin. ([#7509])
+ - build: libsodium configure check fixed to work with newer GCC. ([#7907])
+
+
+### EXPERIMENTAL
+
+ - onchaind: Correctly collect our own (delayed) funds if we have a unilateral close when we are still offering a lease. ([#8111])
+
+
+[#8138]: https://github.com/ElementsProject/lightning/pull/8138
+[#8137]: https://github.com/ElementsProject/lightning/pull/8137
+[#8111]: https://github.com/ElementsProject/lightning/pull/8111
+[#8121]: https://github.com/ElementsProject/lightning/pull/8121
+[#8116]: https://github.com/ElementsProject/lightning/pull/8116
+[#8024]: https://github.com/ElementsProject/lightning/pull/8024
+[#7969]: https://github.com/ElementsProject/lightning/pull/7969
+[#7921]: https://github.com/ElementsProject/lightning/pull/7921
+[#8089]: https://github.com/ElementsProject/lightning/pull/8089
+[#8052]: https://github.com/ElementsProject/lightning/pull/8052
+[#8019]: https://github.com/ElementsProject/lightning/pull/8019
+[#7958]: https://github.com/ElementsProject/lightning/pull/7958
+[#8027]: https://github.com/ElementsProject/lightning/pull/8027
+[#7173]: https://github.com/ElementsProject/lightning/pull/7173
+[#7937]: https://github.com/ElementsProject/lightning/pull/7937
+[#8056]: https://github.com/ElementsProject/lightning/pull/8056
+[#8037]: https://github.com/ElementsProject/lightning/pull/8037
+[#8077]: https://github.com/ElementsProject/lightning/pull/8077
+[#7887]: https://github.com/ElementsProject/lightning/pull/7887
+[#7907]: https://github.com/ElementsProject/lightning/pull/7907
+[#7979]: https://github.com/ElementsProject/lightning/pull/7979
+[#8053]: https://github.com/ElementsProject/lightning/pull/8053
+[#7908]: https://github.com/ElementsProject/lightning/pull/7908
+[#7894]: https://github.com/ElementsProject/lightning/pull/7894
+[#7924]: https://github.com/ElementsProject/lightning/pull/7924
+[#8057]: https://github.com/ElementsProject/lightning/pull/8057
+[#8031]: https://github.com/ElementsProject/lightning/pull/8031
+[#8010]: https://github.com/ElementsProject/lightning/pull/8010
+[#8071]: https://github.com/ElementsProject/lightning/pull/8071
+[#7933]: https://github.com/ElementsProject/lightning/pull/7933
+[#7508]: https://github.com/ElementsProject/lightning/pull/7508
+[#8084]: https://github.com/ElementsProject/lightning/pull/8084
+[#8085]: https://github.com/ElementsProject/lightning/pull/8085
+[#8095]: https://github.com/ElementsProject/lightning/pull/8095
+[#7941]: https://github.com/ElementsProject/lightning/pull/7941
+[#8059]: https://github.com/ElementsProject/lightning/pull/8059
+[#8069]: https://github.com/ElementsProject/lightning/pull/8069
+[#8022]: https://github.com/ElementsProject/lightning/pull/8022
+[#7938]: https://github.com/ElementsProject/lightning/pull/7938
+[#7509]: https://github.com/ElementsProject/lightning/pull/7509
+[#8094]: https://github.com/ElementsProject/lightning/pull/8094
+[#8104]: https://github.com/ElementsProject/lightning/pull/8104
+[#7935]: https://github.com/ElementsProject/lightning/pull/7935
+[#7942]: https://github.com/ElementsProject/lightning/pull/7942
+[#7772]: https://github.com/ElementsProject/lightning/pull/7772
+[#8060]: https://github.com/ElementsProject/lightning/pull/8060
+[25.02]: https://github.com/ElementsProject/lightning/releases/tag/v25.02
+
+
+## [24.11.1] - 2024-12-16: "The lightning-dev Mailing List II"
+
+Minor fixes, particularly for xpay users.
+
+### Changed
+
+ - JSON-RPC: `xpay` now gives the same JSON success return as documented by `pay` when `xpay-handle-pay` is set. ([#7938], [#7952])
+ - JSON-RPC: With `xpay-handle-pay` set, xpay will now be used even if `pay` uses maxfeeprecent or exemptfee parameters (e.g. Zeus) ([#7942])
+ - JSON-RPC: `xpay` now populates more fields, so `listsendpays` and `listpays` show `destination` and `amount_msat` fields for xpay payments. ([#7941])
+ - Logging: we no longer suppress DEBUG messages from subdaemons. ([#7935])
+
+### Fixed
+
+ - JSON-RPC: xpay now works through unannounced channels. ([#7937])
+ - lightning-cli: fix "malformed response" bug ([#7924])
+ - Plugins: `xpay` no longer logs "Got command" at info level. ([#7933])
+ - Build: Alpine/OpenBSD compilation fix  ([#7940])
+
+[#7942]: https://github.com/ElementsProject/lightning/pull/7942
+[#7933]: https://github.com/ElementsProject/lightning/pull/7933
+[#7941]: https://github.com/ElementsProject/lightning/pull/7941
+[#7937]: https://github.com/ElementsProject/lightning/pull/7937
+[#7924]: https://github.com/ElementsProject/lightning/pull/7924
+[#7935]: https://github.com/ElementsProject/lightning/pull/7935
+[#7938]: https://github.com/ElementsProject/lightning/pull/7938
+[#7940]: https://github.com/ElementsProject/lightning/pull/7940
+[#7952]: https://github.com/ElementsProject/lightning/pull/7952
+[24.11.1]: https://github.com/ElementsProject/lightning/releases/tag/v24.11.1
+
+## [24.11] - 2024-12-09: "The lightning-dev Mailing List"
+
+This release named by Dusty Daemon.
+
+### Added
+
+ - JSON-RPC: `listaddresses` to list issued addresses from the node. ([#7800])
+ - Config: `autoconnect-seeker-peers`, allowing seeker to reach out to new nodes for additional gossip (default: 10). ([#7798])
+ - Protocol: `option_quiesce` enabled by default. ([#7586])
+ - JSON-RPC: `fetchinvoice` allows setting invreq_metadata via `payer_metadata` parameter. ([#7786])
+ - hsmtool: generatehsm can run non-interactive, taking options on the cmdline. ([#7102])
+ - Plugins: `pay` now has tracing support for various payment steps. ([#7803])
+ - Plugins: new notification `onionmessage_forward_fail`.
+ - JSON-RPC: `exposesecret` command for encouraging hsm_secret backups. ([#7647])
+ - JSON-RPC: `listpays` has `index`, `start` and `limit` parameters for listing control. ([#7385])
+ - Plugins: bookkeeper has a new RPC `bkpr-editdescriptionbypaymentid` which will update the description for any event with matching payment_id ([#7604])
+ - Config: `grpc-host` option for grpc plugin ([#7479])
+ - JSON-RPC: A new magic `dev-splice` command is added that can take a ‘splice script’ or json payload and perform any complex splice across multiple channels merging the result into a single transaction. Some features are disabled and will be added in time. ([#6980])
+ - JSON-RPC: low-level RPC command `addpsbtinput` to fund PSBTs directly and help with complex splices & dual-opens. ([#6980])
+ - JSON-RPC: `stfu_channels` and `abort_channels` are added for bulk multi-channel splice commands. These allow the user to pause (and resume) multiple channels in place. ([#6980])
+ - JSON-RPC: `injectpaymentonion` for initiating an HTLC like a peer would do. ([#7749])
+ - Documentation: Example documentation on generating custom gRPC ([#7731])
+ - Installation: Nix users can now install CLN from the new flake. ([#7656])
+ - JSON-RPC: `decode` now used modern BOLT 4 language for blinded paths, `first_path_key`. ([#7586])
+ - Plugins: `onion_message_recv` and `onion_message_recv_secret` hooks now used modern BOLT 4 language for blinded paths, `first_path_key`. ([#7586])
+ - JSON-RPC: keysend `maxfee` parameter for consistency with pay. ([#7227]) ([#7653])
+ - `hsmtool`: `getnodeid` command derives the node id from the hsm_secret, to verify it's the correct secret. ([#7644])
+
+
+### Changed
+
+ - Plugins: grpc now starts on port 9736 by default (localhost, see `grpc-host`) ([#7479])
+ - Config: bolt12 now enabled by default (finally!) ([#7833])
+ - Protocol: we now connect to additional nodes for improved gossip (see `autoconnect-seeker-peers`) ([#7798])
+ - Protocol: Own-channel gossip is broadcast to more peers (up to 50, not 5). ([#7873])
+ - Build: Added architecture identifier to Ubuntu release ([#7797])
+ - Logging: connectd now logs unknown messages as "UNKNOWN" not "INVALID" to avoid freaking people out. ([#7892])
+ - Protocol: we now create a low-priority (2016 down to 12 blocks fee target) anchor for low-fee unilateral closes even if there's no urgency. ([#7832])
+ - Protocol: splicing moved from test numbers to spec numbers. ([#7719])
+ - Protocol: Support added for peers that wish to rotate their funding pubkey during a splice. ([#7719])
+ - Startup: reconnecting to peers at startup should be significantly faster (dependent on machine speed). ([#7630])
+ - Protocol: we remember the last successful address we connected to for important peers. ([#7630])
+ - Protocol: Gossipd requests a full sync from a random peer every hour. ([#7768])
+ - JSON-RPC: `injectonionmessage` API simplified and documented.
+ - JSON-RPC: Improved error messaging for splice commands. ([#7719])
+ - JSON-RPC: built-in plugins can now be stopped using "plugin stop". ([#7799])
+ - Wallet: Taproot addresses are used for unilateral-close change addresses. ([#7800])
+ - JSON-RPC: `close` now outputs txs & txids of all closing transactions (splice candidates can cause there to be multiple). ([#7466])
+ - JSON-RPC: `splice_update` can in some cases now return the remotely partiall signed psbt to the user, if so `signtures_secured` will be true. ([#6980])
+ - JSON-RPC: `splice_signed` parameters are switched in order to make `channel_id` an optional parameter, enabling multi-splice-signatures. ([#6980])
+ - `pay`: Discarding an overly long or expensive route does not blocklist channels anymore. ([#7494])
+ - Plugins: `cln-grpc` Upgrade tonic version and ([#7598])
+
+
+### Deprecated
+
+Note: You should always set `allow-deprecated-apis=false` to test for changes.
+
+ - Config: `experimental-offers` (it's now the default). ([#7833])
+ - Config: `experimental-quiesce`: it's now the default. ([#7586])
+ - JSON-RPC: `close` `tx` and `txid` field (use `txs` and `txids`) ([#7466])
+ - JSON-RPC: `decodepay`: use `decode`. ([#7551])
+ - JSON-RPC: `decode` `blinding` in blinded path: use `first_path_key`. ([#7586])
+ - JSON-RPC: `onion_message_recv` and `onion_message_recv_secret` hooks `blinding` in blinded path: use `first_path_key`. ([#7586])
+
+
+### Fixed
+
+ - JSON-RPC: `decode` refused to decode expired bolt12 invoices. ([#7890])
+ - JSON-RPC: `listforwards` `received-time` is always present (it could be missing for ancient nodes, now it will be 0) ([#7744])
+ - Plugins: `cln-grpc` now understands channel type `anchors/even` ([#7628])
+ - Plugins: `cln-grpc` no longer logs a warning if a notification does not have a handler ([#7867])
+ - JSON-RPC: `close` now correctly reports the txid of the remote onchain unilateral tx if it races with a peer close. ([#7593])
+ - Protocol: we no longer try to spend anchors if a commitment tx is already mined (reported by @niftynei). ([#7593])
+ - pyln-client: plugins now compatible with CLN <= 24.05 (broken in 24.08) ([#7852])
+ - Protocol: Receiving bolt12 payments where we have no public channels would fail a few blocks after startup. ([#7839])
+ - Protocol: entry to blinded paths returns more useful errors (e.g if it's the final node, you get a real error, otherwise you get invalid_onion_blinding). ([#7839])
+ - build: fix overzealous warning from clang 19. ([#7829])
+ - `gossipd` will no longer miss some channel closes on restart. ([#7343])
+ - `onchaind` can miss conclusion of final txs in some cases, will now replay independently. ([#7343])
+ - Plugins: `autoclean` is now gentler on the node when doing giant cleans. ([#7805])
+ - Plugins: libplugin plugins can now intercept `rpc_command` hook without deadlocking. ([#7750])
+ - Plugins: `pay`: debug logging now uses correct JSON ids. ([#7750])
+ - Tools: Fixed regtest directory location for giantnode script. ([#7791])
+ - JSON-RPC: `fetchinvoice` typo fixed about parameters name in err str ([#7787])
+ - Performance: `connectd` and `gossipd` message queues are much more efficient. ([#7767])
+ - Documentation: version strings in documentation should now be correvt ([#7694])
+ - Protocol: we could get confused on restart and not re-transmit our own channel_updates. ([#7737])
+ - connectd: crash on erroneous timeout. ([#7736])
+ - Performance: `pay` now only calls listpeerchannels once ([#7705])
+ - gossipd: crash errors with large gossip_store (>4MB) growth on longer-running nodes. ([#7729])
+ - Performance: `pay` pathfinding speedups for large nodes. ([#7726])
+ - Performance: `listchannels` performance improved on large nodes if deprecated options disabled (omitting call to `listpeerchannels`) ([#7704])
+ - pyln-testing: Fix file descriptor leak in bitcoind fixture. ([#7130]) ([#7669])
+ - JSON-RPC: `listpeerchannels` (and thus, pay) sped up on very large nodes. ([#7679])
+ - lightningd: no longer crash if a plugin dies during lightningd startup. ([#7673])
+ - cln-plugin: Change default log level filter back to INFO ([#7668])
+ - Logging: removed bogus "**BROKEN** plugin-topology: DEPRECATED API USED: listchannels.include_private" message. ([#7663])
+ - Logging: When DEBUG printing first tx depth, we printed the wrong value ([#7910])
+ - Documentation: schemas: Make description in `Wait(any)invoiceResponse` optional to handle BOLT12 ([#7667])
+ - Fixed intermittant bug where hsmd (particularly, but also lightningd) could use 100% CPU. ([#7661])
+ - Docker image created via github actions correctly reads the tag available on the HEAD. ([#7625])
+ - Build: Core lightning's version will not be suffixed with -modded anymore. ([#7625])
+
+
+### EXPERIMENTAL
+
+ - Plugins: `cln-xpay`, with associated `xpay` command for payments (plans to replace `pay` in future). ([#7799])
+ - Config: option `xpay-handle-pay` can be used to call xpay when pay is used in many cases (but output is different from pay!) ([#7799])
+ - Plugins: `askrene` which provides `getroutes` and a complete API for adding information in layers. ([#7782])
+
+
+[#7694]: https://github.com/ElementsProject/lightning/pull/7694
+[#7644]: https://github.com/ElementsProject/lightning/pull/7644
+[#7803]: https://github.com/ElementsProject/lightning/pull/7803
+[#7782]: https://github.com/ElementsProject/lightning/pull/7782
+[#7479]: https://github.com/ElementsProject/lightning/pull/7479
+[#7705]: https://github.com/ElementsProject/lightning/pull/7705
+[#7839]: https://github.com/ElementsProject/lightning/pull/7839
+[#7551]: https://github.com/ElementsProject/lightning/pull/7551
+[#7798]: https://github.com/ElementsProject/lightning/pull/7798
+[#7647]: https://github.com/ElementsProject/lightning/pull/7647
+[#7744]: https://github.com/ElementsProject/lightning/pull/7744
+[#7832]: https://github.com/ElementsProject/lightning/pull/7832
+[#7767]: https://github.com/ElementsProject/lightning/pull/7767
+[#7768]: https://github.com/ElementsProject/lightning/pull/7768
+[#7656]: https://github.com/ElementsProject/lightning/pull/7656
+[#7604]: https://github.com/ElementsProject/lightning/pull/7604
+[#7707]: https://github.com/ElementsProject/lightning/pull/7707
+[#7750]: https://github.com/ElementsProject/lightning/pull/7750
+[#7628]: https://github.com/ElementsProject/lightning/pull/7628
+[#7343]: https://github.com/ElementsProject/lightning/pull/7343
+[#7102]: https://github.com/ElementsProject/lightning/pull/7102
+[#7731]: https://github.com/ElementsProject/lightning/pull/7731
+[#7679]: https://github.com/ElementsProject/lightning/pull/7679
+[#7673]: https://github.com/ElementsProject/lightning/pull/7673
+[#7749]: https://github.com/ElementsProject/lightning/pull/7749
+[#7668]: https://github.com/ElementsProject/lightning/pull/7668
+[#7494]: https://github.com/ElementsProject/lightning/pull/7494
+[#7786]: https://github.com/ElementsProject/lightning/pull/7786
+[#7466]: https://github.com/ElementsProject/lightning/pull/7466
+[#7787]: https://github.com/ElementsProject/lightning/pull/7787
+[#7625]: https://github.com/ElementsProject/lightning/pull/7625
+[#7800]: https://github.com/ElementsProject/lightning/pull/7800
+[#7829]: https://github.com/ElementsProject/lightning/pull/7829
+[#6980]: https://github.com/ElementsProject/lightning/pull/6980
+[#7653]: https://github.com/ElementsProject/lightning/pull/7653
+[#7734]: https://github.com/ElementsProject/lightning/pull/7734
+[#7867]: https://github.com/ElementsProject/lightning/pull/7867
+[#7630]: https://github.com/ElementsProject/lightning/pull/7630
+[#7667]: https://github.com/ElementsProject/lightning/pull/7667
+[#7799]: https://github.com/ElementsProject/lightning/pull/7799
+[#7736]: https://github.com/ElementsProject/lightning/pull/7736
+[#7719]: https://github.com/ElementsProject/lightning/pull/7719
+[#7593]: https://github.com/ElementsProject/lightning/pull/7593
+[#7833]: https://github.com/ElementsProject/lightning/pull/7833
+[#7770]: https://github.com/ElementsProject/lightning/pull/7770
+[#7737]: https://github.com/ElementsProject/lightning/pull/7737
+[#7586]: https://github.com/ElementsProject/lightning/pull/7586
+[#7726]: https://github.com/ElementsProject/lightning/pull/7726
+[#7805]: https://github.com/ElementsProject/lightning/pull/7805
+[#7598]: https://github.com/ElementsProject/lightning/pull/7598
+[#7791]: https://github.com/ElementsProject/lightning/pull/7791
+[#7385]: https://github.com/ElementsProject/lightning/pull/7385
+[#7669]: https://github.com/ElementsProject/lightning/pull/7669
+[#7663]: https://github.com/ElementsProject/lightning/pull/7663
+[#7661]: https://github.com/ElementsProject/lightning/pull/7661
+[#7729]: https://github.com/ElementsProject/lightning/pull/7729
+[#7852]: https://github.com/ElementsProject/lightning/pull/7852
+[#7740]: https://github.com/ElementsProject/lightning/pull/7740
+[#7704]: https://github.com/ElementsProject/lightning/pull/7704
+[#7756]: https://github.com/ElementsProject/lightning/pull/7756
+[#7873]: https://github.com/ElementsProject/lightning/pull/7873
+[#7890]: https://github.com/ElementsProject/lightning/pull/7890
+[#7797]: https://github.com/ElementsProject/lightning/pull/7797
+[#7892]: https://github.com/ElementsProject/lightning/pull/7892
+[#7910]: https://github.com/ElementsProject/lightning/pull/7910
+[24.11]: https://github.com/ElementsProject/lightning/releases/tag/v24.11
+
+
+## [24.08.2] - 2024-10-18: "Steel Backed-up Channels"
+
+This point release addresses few crash fixes and includes an enhancement to remember and update channel hints for payments.
+
+### Changed
+
+- pay: Now remembers and updates channel hints across payments ([#7494])
+- pay: Discarding an overly long or expensive route does not blacklist channels anymore. ([#7494])
+- grpc: Channel type `anchors/even` was added to the grpc bindings. ([#7628])
+
+### Fixed
+- gossipd: crash errors with large gossip_store (>4MB) growth on longer-running nodes. ([#7729])
+- connectd: crash on erroneous timeout. ([#7736])
+- Protocol: we could get confused on restart and not re-transmit our own channel_updates. ([#7737])
+- rpc: `listpeerchannels` (and thus, pay) sped up on very large nodes. ([#7679])
+- Improved pathfinding speed for large nodes. ([#7726])
+
+
+[#7494]: https://github.com/ElementsProject/lightning/pull/7494
+[#7628]: https://github.com/ElementsProject/lightning/pull/7628
+[#7679]: https://github.com/ElementsProject/lightning/pull/7679
+[#7726]: https://github.com/ElementsProject/lightning/pull/7726
+[#7729]: https://github.com/ElementsProject/lightning/pull/7729
+[#7736]: https://github.com/ElementsProject/lightning/pull/7736
+[#7737]: https://github.com/ElementsProject/lightning/pull/7737
+[24.08.2]: https://github.com/ElementsProject/lightning/releases/tag/v24.08.2
+
+
+## [24.08.1] - 2024-09-16: "Steel Backed-up Channels"
+
+This hotfix release fixes few crash issues and some other bugs.
+
+### Fixed
+
+ - No longer crash if a plugin dies during lightningd startup. ([#7673])
+ - Fixed crash when starting channeld for older channel with no local alias. ([#7664])
+ - Will not crash for older pre-tlv peers where send_tlvs is NULL. ([#7650])
+ - Compilation will fail gracefully if `jq` is missing or an outdated version is detected. ([#7662])
+ - cln-plugin: Change default log level filter back to INFO ([#7668])
+ - Fixed intermittant bug where hsmd (particularly, but also lightningd) could use 100% CPU. ([#7661])
+ - Made `description` optional in `Wait(any)invoiceResponse` to handle BOLT12 ([#7667])
+ - Removed bogus "**BROKEN** plugin-topology: DEPRECATED API USED: listchannels.include_private" message. ([#7663])
+
+
+[#7673]: https://github.com/ElementsProject/lightning/pull/7673
+[#7668]: https://github.com/ElementsProject/lightning/pull/7668
+[#7667]: https://github.com/ElementsProject/lightning/pull/7667
+[#7664]: https://github.com/ElementsProject/lightning/pull/7664
+[#7663]: https://github.com/ElementsProject/lightning/pull/7663
+[#7662]: https://github.com/ElementsProject/lightning/pull/7662
+[#7661]: https://github.com/ElementsProject/lightning/pull/7661
+[#7650]: https://github.com/ElementsProject/lightning/pull/7650
+
+
+## [24.08] - 2024-08-28: "Steel Backed-up Channels"
+
+This release named by @Lagrang3.
+
+### Added
+
+ - protocol: onion messages are now supported by default. ([#7455])
+ - protocol: onion messages can now be forwarded by short_channel_id. ([#7455])
+ - pay: The pay plugin now checks whether we have enough spendable capacity before computing a route, returning a clear error message if we don't ([#7418])
+ - pay: Payments now emit `channel_hint_updated` notification to share inferred balances and observations across multiple payments. ([#7487])
+ - pay: The pay plugin now returns better error codes ([#7418])
+ - reckless-rpc plugin: issue commands to reckless over rpc. ([#7506])
+ - reckless: accepts json array input for command targets ([#7484])
+ - reckless provides json output with option flag -j/--json ([#7484])
+ - reckless: added the ability to install rust plugins. ([#7484])
+ - JSON-RPC: `listpeers` `features` array string uses "option_anchors" for feature 22/23, following renaming in BOLT 9. ([#7388])
+ - plugins: `bookkeeper` now listens for two custom events: `utxo_deposit` and `utxo_spend`. This allows for 3rd party plugins to send onchain coin events to the `bookkeeper`.  See the new plugins/bkpr/README.md for details on how these work! ([#7258])
+ - plugins: Add payment_id parameter to bkpr-listaccountevents to filter events. ([#7536])
+ - cln-plugin: add multi options for String and i64 ([#7544])
+ - pyln-client: Added a notification mechanism for config changes ([#7289])
+ - pyln-client: implement setconfig hook for plugins so you can see changes in `dynamic` options. ([#7289])
+ - doc: all examples in the documentation are now generated from running the code, so they are current. ([#7457])
+
+
+### Changed
+
+ - protocol: We now send current peers our changed gossip (even if they set timestamp_filter otherwise), not just on reconnect. ([#7554])
+ - protocol: we now always ask the first peer for all its gossip. ([#7512])
+ - connectd: I/O optimizations to significantly speed up larger nodes. ([#7365])
+ - plugins: the `fetchinvoice` plugin has been combined into the `offers` plugin. ([#7456])
+ - close: We no longer attempt to publish a unilateral close that'd fail anyway when we witness a close onchain. ([#7447])
+ - pay: Improved logging should facilitate debugging considerably. ([#7418])
+ - reckless: option flags are now position independent. ([#7484])
+ - plugins: now allows date and time sqlite functions. ([#7467])
+ - splice: added outnum of new funding output to splice_signed RPC command ([#7465])
+ - lightningd: we wait for bitcoind if it has somehow gone backwards (as long as header height is still ok). ([#7342])
+ - wallet: The channel status is printed when loading it from the DB ([#7354])
+ - JSON-RPC: `listclosedchannels`, `listpeerchannels`, `openchannel_update`, `openchannel_init`, `fundchannel`, `fundchannel_start` and `multifundchannel`: `channel_type` array `names` now contains "anchors" instead of "anchors_zero_fee_htlc_tx". ([#7388])
+ - JSON-RPC: Do not return the contents of invalid parameters in error messages, refer to logs (use 'check' to get full error messages) ([#7420])
+ - lightningd: `--list-features-only` now lists "option_anchors" instead of "option_anchors_zero_fee_htlc_tx". ([#7388])
+ - updated Bitcoin to v27.1 and Elements to v23.2.1 ([#7436])
+ - update libwally to 1.3.0 ([#7480])
+
+
+### Deprecated
+
+Note: You should always set `allow-deprecated-apis=false` to test for changes.
+
+ - JSON-RPC: `listpeers` `features` array string "option_anchors_zero_fee_htlc_tx": use "option_anchors" (spec renamed it). ([#7388])
+ - config: the --experimental-onion-messages option is ignored (on by default). ([#7455])
+ - config: the --experimental-anchors option is ignored (on by default since v24.02). ([#7454])
+ - pyln-client: `category`, `description` and `long_description` for RPC commands are deprecated now. ([#7520])
+
+
+### Removed
+
+ - JSON-RPC: `sendonionmessage` (was experimental only, use `injectonionmessage`) ([#7461])
+ - JSON-RPC: `autocleaninvoice` command (deprecated v22.11, EOL v24.02) ([#7298])
+ - plugins: `estimatefees` returning feerates by name (e.g. "opening"); deprecated in v23.05. ([#7380])
+
+
+### Fixed
+
+ - protocol: we can now open unannounced channels with LND nodes again. ([#7564])
+ - plugins: `sql` crash on querying `listpeerchannels` during channel establishment. ([#7578])
+ - config: whitespace at the end of (most) options is now ignored, not complained about. ([#7251])
+ - connectd: now should use far less CPU on large nodes. ([#7365])
+ - lightningd: occasionally we could miss transaction outputs (not telling gossipd, or even onchaind) ([#7567])
+ - notifications: we now send a `coin_movement` notification for splice confirmations of channel funding outpoint spends. ([#7533])
+
+### EXPERIMENTAL
+
+ - JSON-RPC: `offer` removed `@` prefix support from `recurrence_base` (use `recurrence_start_any_period` set to `false`) ([#7380])
+ - protocol: pay can now pay to bolt12 invoices if entry to blinded hop is specified as a short_channel_id (rather than node id). ([#7461])
+ - plugins: pay can now pay a bolt12 invoice even if we, ourselves, are the head of the blinded path within it. ([#7461])
+ - offers: automatically add a blinded path from a peer if we have no public channels, so unannounced nodes can have offers too. ([#7461])
+ - offers: we can now self-fetch and self-pay BOLT12 offers and invoices. ([#7461])
+ - offers: recurring offers had incompatible changes, won't work against older versions. ([#7476])
+ - offers: handle experimental ranges in offers/invoice_requests/invoices. ([#7474])
+ - offers: `invoicerequest` will set a blinded path if we're an unannounced node. ([#7476])
+ - offers: `sendinvoice` will use a blinded path in an invoice_request, if specified. ([#7476])
+ - offers: maintain unknown fields offers/invoice_requests correctly. ([#7474])
+ - offers: fixed: onionmessage replies now work even if we need to route to the start of the blinded reply path. ([#7456])
+ - offers: fixed: fetchinvoice tries all blinded paths until one is usable, and handles case where we have to route more than one hop to reach the entry point. ([#7456])
+ - renepay: prune the network by disabling channels we don't like, eg. very low max_htlc. ([#7403])
+ - renepay: fixed: un-reserve routes that have completed or failed ([#7357])
+ - renepay: Add a dev parameter representing a constant probability of availability for all channels in the network. ([#7540])
+ - renepay: add cli option "exclude" to manually disable channels and nodes. ([#7403])
+
+
+
+[#7540]: https://github.com/ElementsProject/lightning/pull/7540
+[#7578]: https://github.com/ElementsProject/lightning/pull/7578
+[#7447]: https://github.com/ElementsProject/lightning/pull/7447
+[#7420]: https://github.com/ElementsProject/lightning/pull/7420
+[#7418]: https://github.com/ElementsProject/lightning/pull/7418
+[#7454]: https://github.com/ElementsProject/lightning/pull/7454
+[#7436]: https://github.com/ElementsProject/lightning/pull/7436
+[#7484]: https://github.com/ElementsProject/lightning/pull/7484
+[#7544]: https://github.com/ElementsProject/lightning/pull/7544
+[#7476]: https://github.com/ElementsProject/lightning/pull/7476
+[#7456]: https://github.com/ElementsProject/lightning/pull/7456
+[#7403]: https://github.com/ElementsProject/lightning/pull/7403
+[#7289]: https://github.com/ElementsProject/lightning/pull/7289
+[#7388]: https://github.com/ElementsProject/lightning/pull/7388
+[#7474]: https://github.com/ElementsProject/lightning/pull/7474
+[#7536]: https://github.com/ElementsProject/lightning/pull/7536
+[#7457]: https://github.com/ElementsProject/lightning/pull/7457
+[#7467]: https://github.com/ElementsProject/lightning/pull/7467
+[#7487]: https://github.com/ElementsProject/lightning/pull/7487
+[#7251]: https://github.com/ElementsProject/lightning/pull/7251
+[#7365]: https://github.com/ElementsProject/lightning/pull/7365
+[#7461]: https://github.com/ElementsProject/lightning/pull/7461
+[#7520]: https://github.com/ElementsProject/lightning/pull/7520
+[#7357]: https://github.com/ElementsProject/lightning/pull/7357
+[#7480]: https://github.com/ElementsProject/lightning/pull/7480
+[#7465]: https://github.com/ElementsProject/lightning/pull/7465
+[#7354]: https://github.com/ElementsProject/lightning/pull/7354
+[#7512]: https://github.com/ElementsProject/lightning/pull/7512
+[#7380]: https://github.com/ElementsProject/lightning/pull/7380
+[#7564]: https://github.com/ElementsProject/lightning/pull/7564
+[#7455]: https://github.com/ElementsProject/lightning/pull/7455
+[#7342]: https://github.com/ElementsProject/lightning/pull/7342
+[#7506]: https://github.com/ElementsProject/lightning/pull/7506
+[#7554]: https://github.com/ElementsProject/lightning/pull/7554
+[#7258]: https://github.com/ElementsProject/lightning/pull/7258
+[#7533]: https://github.com/ElementsProject/lightning/pull/7533
+[#7567]: https://github.com/ElementsProject/lightning/pull/7567
+[#7298]: https://github.com/ElementsProject/lightning/pull/7298
+[24.08]: https://github.com/ElementsProject/lightning/releases/tag/v24.08
+
+
+
+## [24.05] - 2024-06-04: "The Infinitely Divisible Satoshi"
+
+This release named by @daywalker90.
+
+### Added
+
+ - JSON-RPC: `createrune` new restriction `pinv` to examine bolt11/bolt12 invoice fields (e.g. amount of invoice). ([#7165])
+ - Plugins: `cln-plugin` adds dynamic configs and a callback for changes ([#7293])
+ - JSON-RPC: `pay` has a new parameter `partial_msat` to only pay part of an invoice (someone else presumably will pay the rest at the same time!) ([#7145])
+ - JSON-RPC: `check` `keysend` now checks with HSM that it will approve it. ([#7111])
+ - Plugins: Can now opt in to handle `check` command on their commands, for more thorough checking. ([#7111])
+ - JSON-RPC: `check` `setconfig` now checks that the new config setting would be valid. ([#7111])
+ - JSON-RPC: `check` `setconfig` on plugin options can now check the config value would be accepted. ([#7111])
+ - Plugins: `cln-grpc` adds notifications over the grpc interface. Configurable with config parameter `grpc-msg-buffer-size`. ([#7084])
+ - Plugins: Added `wss-proxy`, a WSS Proxy server with `wss-bind-addr` and `wss-certs` configurations. ([#7225])
+ - Plugins: `cln-grpc` added GRPC support for remaining methods: `dev-forget-channel`, `emergencyrecover`, `recover`, `recoverchannel`, `funderupdate`, `help`, `invoicerequest`, `listinvoicerequests`, `disableinvoicerequest`, `listconfigs`, `makesecret`, `multiwithdraw`, `showrunes`, `createrune`, `blacklistrune`, `checkrune` ([#7317]), addpsbtoutput ([#7108]), `openchannel_init`, `openchannel_abort`, `openchannel_bump`, `openchannel_signed`, `openchannel_update` ([#7230]), `delpay` ([#7232]), `delforward` ([#7260]), `autoclean-once`, `autoclean-status` ([#7238]), `fundchannel_start`, `fundchannel_complete`, `fundchannel_cancel` ([#7231]), `bkpr-channelsapy`, `bkpr-dumpincomecsv`, `bkpr-inspect`, `bkpr-listaccountevents`, `bkpr-listbalances` ([#7256]), `disableoffer` ([#7233]), `parsefeerate`, `plugin`, `renepay`, `renepaystatus`, `sendinvoice` ([#7272]), `reserveinputs`, `unreserveinputs`, `splice_init`, `splice_signed`, `splice_update` ([#7273]), `sendonionmessage`, `setconfig`, `setpsbtversion`, `upgradewallet` ([#7274]).
+ - Plugins: `cln_plugin` adds rust plugin support for wildcard `*` subscriptions.  ([#7106])
+ - Config: Add `bitcoin-rpcclienttimeout` config parameter. ([#7095])
+ - Plugins: new `log` notification when a log line is emitted. ([#6990])
+ - Config: new log level `trace` where we moved the very noisiest `debug` logs. ([#7280])
+ - Plugins: `clnrest` added a new configuration `clnrest-swagger-root` to change the default Swagger UI path from `/` to custom url. ([#7256])
+
+
+### Changed
+
+ - Documentation: great documentation rewrite, all reference pages now generated from the fully-tested JSON schemas and include examples. ([#6995])
+ - Protocol: `--ignore-fee-limits` / `setchannel ignorefeelimits` no longer applies to mutual close. ([#7252])
+ - Plugins: `bcli`: Add a path that tries to fetch blocks ([#7240])
+ - Plugins: libplugin now shows plugin option default values (where they're non-trivial) ([#7306])
+ - Runes: named parameters (e.g. `pnameamountmsat`) no longer need to remove underscores (i.e. `pnameamount_msat` now works as expected). ([#7124])
+ - lightningd: we now try to increase the number of file descriptors, if it's less than twice the number of channels at startup (and log if we cannot!). ([#7237])
+ - connectd: prioritize peers with channels (and log!) if we run low on file descriptors. ([#7237])
+ - lightningd: Processing blocks should now be faster ([#7101])
+ - Plugins: `cln-grpc` adds routes to `decode` and `decodepay` results ([#7317])
+ - hsmd: the hsmd now supports `HSM_VERSION 6` ([#7178])
+ - hsmd: `HSM_VERSION 6`: `get_per_commitment_point` does not imply index - 2 is revoked, makes it safe to call on any index. ([#7178])
+ - Documentation: Merged `example_json_request` and `example_json_response` in a single `json_examples` array to maintain the request and its corresponding response together. ([#7181])
+ - JSON-RPC: `stop` and `recover` now return a JSON object (not a raw string!) like every other command does. ([#6995])
+ - Plugins: `pay` payments are more robust for nodes that are currently syncing. ([#7190])
+
+
+### Deprecated
+
+Note: You should always set `allow-deprecated-apis=false` to test for changes.
+
+
+
+### Removed
+
+ - Plugins: no longer allow missing `id` field in commando requests (deprecated v23.02, EOL v24.02) ([#7094])
+ - JSON-RPC: `createrune` restrictions as raw strings (use arrays) (deprecated v23.05, EOL 24.02). ([#7094])
+ - JSON-RPC: `listpeers` `channels` (deprecated v23.02, EOL v24.02) ([#7094])
+ - JSON-RPC: `sendpay` ignoring first channel (deprecated v0.12, EOL v24.02) ([#7094])
+ - Config: `experimental-websocket-port` (deprecated 23.08, EOL 24.02) ([#7094])
+ - Plugins: `funding_locked` from `channel_opened` notification (deprecated v22.11, EOL v24.02) ([#7094])
+ - JSON-RPC: `feerates` output fields `delayed_to_us` and `htlc_resolution`. ([#7094])
+ - Config: `autocleaninvoice-cycle` and `autocleaninvoice-expired-by` (deprecated v22.11, EOL v24.02) ([#7094])
+ - JSON-RPC: `delexpiredinvoice` (deprecated v22.11, EOL v24.02) ([#7094])
+ - JSON-RPC feerates by internal names ("opening", "mutual_close", "delayed_to_us", "htlc_resolution", "penalty", "min_acceptable", "max_acceptable") (deprecated v23.05, EOL v24.02). ([#7094])
+ - Plugins: `invoice_payment` and `htlc_accepted` hook `failure_code` response (deprecated v22.08 and v0.8, EOL v23.02) ([#7094])
+
+
+### Fixed
+
+ - Plugins: `pay` now correctly estimates channel capacity ([#7188])
+ - lightningd: avoid crash on signing failure when trying to spend anchor outputs. ([#7291])
+ - Plugins: `renepay` fixed a race condition leading to a crash. ([#7125])
+ - JSON-RPC: `fundchannel_start` now disallows a non-zero `mindepth` parameter if you ask for a zeroconf `channel_type`. ([#7175])
+ - pyln-client: Fix Plugin.notify_message() not to ignore `level` parameter. ([#7287])
+ - JSON-RPC: `multifundchannel` with `all` as an amount works as expected. ([#7037])
+ - Plugins: `pay` crash fixed, caused by parsing uncommitted dual open channels ([#7235])
+ - Plugins: `clnrest` now correctly self-disables if Python not present at all. ([#7211])
+ - lightningd: slow memory leak when using plugin hooks fixed (introduced in v23.11) ([#7192])
+ - Plugins: `recovery` is less noisy. ([#7116])
+ - Plugins: `renepay` handles htlc_max correctly for local channels. ([#7159])
+ - Plugins: The recover plugin now avoids trying to recover closed channels. ([#7216])
+ - Gossmap: Avoid adding redundant channel announcements to the gossip_store. ([#7330])
+ - Protocol: forward legacy non-TLV onions which we removed in 22.11 and spec itself in Feb 2022.  Still sent by LND nodes who haven't seen our node_announcement. ([#7352])
+ - Protocol: we once again send CHANNEL_REESTABLISH responses on closing channels. ([#7353])
+ - gossipd: Fixed a crash when processing pending node announcements. ([#7368])
+
+
+### EXPERIMENTAL
+
+ - offers: We will now reply to invoice_request messages even if reply path requires us to make an outgoing connection (LDK does this) ([#7304])
+ - offers: we now understand blinded paths which use a short-channel-id(+direction) as entry point. ([#7212])
+ - offers: Fix blinded paths in invoices - use node_id and set final node's CLTV delta. ([#7311])
+
+
+
+[#7368]: https://github.com/ElementsProject/lightning/pull/7368
+[#7353]: https://github.com/ElementsProject/lightning/pull/7353
+[#7352]: https://github.com/ElementsProject/lightning/pull/7352
+[#7159]: https://github.com/ElementsProject/lightning/pull/7159
+[#7116]: https://github.com/ElementsProject/lightning/pull/7116
+[#7230]: https://github.com/ElementsProject/lightning/pull/7230
+[#7232]: https://github.com/ElementsProject/lightning/pull/7232
+[#7260]: https://github.com/ElementsProject/lightning/pull/7260
+[#7238]: https://github.com/ElementsProject/lightning/pull/7238
+[#7231]: https://github.com/ElementsProject/lightning/pull/7231
+[#7256]: https://github.com/ElementsProject/lightning/pull/7256
+[#7233]: https://github.com/ElementsProject/lightning/pull/7233
+[#7190]: https://github.com/ElementsProject/lightning/pull/7190
+[#7095]: https://github.com/ElementsProject/lightning/pull/7095
+[#7272]: https://github.com/ElementsProject/lightning/pull/7272
+[#7273]: https://github.com/ElementsProject/lightning/pull/7273
+[#7311]: https://github.com/ElementsProject/lightning/pull/7311
+[#7274]: https://github.com/ElementsProject/lightning/pull/7274
+[#7330]: https://github.com/ElementsProject/lightning/pull/7330
+[#7181]: https://github.com/ElementsProject/lightning/pull/7181
+[#7124]: https://github.com/ElementsProject/lightning/pull/7124
+[#7287]: https://github.com/ElementsProject/lightning/pull/7287
+[#6995]: https://github.com/ElementsProject/lightning/pull/6995
+[#7225]: https://github.com/ElementsProject/lightning/pull/7225
+[#7317]: https://github.com/ElementsProject/lightning/pull/7317
+[#7108]: https://github.com/ElementsProject/lightning/pull/7108
+[#7111]: https://github.com/ElementsProject/lightning/pull/7111
+[#7240]: https://github.com/ElementsProject/lightning/pull/7240
+[#7165]: https://github.com/ElementsProject/lightning/pull/7165
+[#7175]: https://github.com/ElementsProject/lightning/pull/7175
+[#7212]: https://github.com/ElementsProject/lightning/pull/7212
+[#7252]: https://github.com/ElementsProject/lightning/pull/7252
+[#7094]: https://github.com/ElementsProject/lightning/pull/7094
+[#7145]: https://github.com/ElementsProject/lightning/pull/7145
+[#7101]: https://github.com/ElementsProject/lightning/pull/7101
+[#6990]: https://github.com/ElementsProject/lightning/pull/6990
+[#7178]: https://github.com/ElementsProject/lightning/pull/7178
+[#7188]: https://github.com/ElementsProject/lightning/pull/7188
+[#7306]: https://github.com/ElementsProject/lightning/pull/7306
+[#7037]: https://github.com/ElementsProject/lightning/pull/7037
+[#7304]: https://github.com/ElementsProject/lightning/pull/7304
+[#7280]: https://github.com/ElementsProject/lightning/pull/7280
+[#7226]: https://github.com/ElementsProject/lightning/pull/7226
+[#7291]: https://github.com/ElementsProject/lightning/pull/7291
+[#7235]: https://github.com/ElementsProject/lightning/pull/7235
+[#7192]: https://github.com/ElementsProject/lightning/pull/7192
+[#7293]: https://github.com/ElementsProject/lightning/pull/7293
+[#7211]: https://github.com/ElementsProject/lightning/pull/7211
+[#7237]: https://github.com/ElementsProject/lightning/pull/7237
+[#7256]: https://github.com/ElementsProject/lightning/pull/7256
+[24.05]: https://github.com/ElementsProject/lightning/releases/tag/v24.05
+
+
+
 ## [24.02.1] - 2024-03-08: "uint needs signature"
 
 This release named by Erik de Smedt (@ErikDeSmedt).

Cargo.toml

@@ -4,8 +4,10 @@ strip = "debuginfo"
 [workspace]
 resolver = "2"
 members = [
-  "cln-rpc",
-  "cln-grpc",
-  "plugins",
-  "plugins/grpc-plugin",
+    "cln-rpc",
+    "cln-grpc",
+    "plugins",
+    "plugins/grpc-plugin",
+    "plugins/rest-plugin",
+    "plugins/lsps-plugin",
 ]

Dockerfile

@@ -1,26 +1,35 @@
-# This dockerfile is meant to compile a core-lightning x64 image
-# It is using multi stage build:
-# * downloader: Download litecoin/bitcoin and qemu binaries needed for core-lightning
-# * builder: Compile core-lightning dependencies, then core-lightning itself with static linking
-# * final: Copy the binaries required at runtime
-# The resulting image uploaded to dockerhub will only contain what is needed for runtime.
-# From the root of the repository, run "docker build -t yourimage:yourtag ."
+# This Dockerfile is used by buildx to build ARM64, AMD64, and ARM32 Docker images from an AMD64 host.
+# To speed up the build process, we are cross-compiling rather than relying on QEMU.
+# There are four main stages:
+# * downloader: Downloads specific binaries needed for core lightning for each architecture.
+# * builder: Cross-compiles for each architecture.
+# * builder-python: Builds Python dependencies for wss-proxy with QEMU.
+# * final: Creates the runtime image.
 
-FROM --platform=$BUILDPLATFORM debian:bullseye-slim as base-downloader
+ARG DEFAULT_TARGETPLATFORM="linux/amd64"
+ARG BASE_DISTRO="debian:bookworm-slim"
+
+FROM --platform=$BUILDPLATFORM ${BASE_DISTRO} AS base-downloader
 RUN set -ex \
 	&& apt-get update \
-	&& apt-get install -qq --no-install-recommends ca-certificates dirmngr wget
+	&& apt-get install -qq --no-install-recommends ca-certificates dirmngr wget qemu-user-static binfmt-support
 
-FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-amd64
+FROM base-downloader AS base-downloader-linux-amd64
 ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu
+ENV DESCHASHPLUGIN_ARCH=linux-amd64
+ENV DESCHASHPLUGIN_HASH=deadc00c68fac80b2718d92f69bf06acd8fff646228d497bbb76a4f0a12ca217
 
-FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-arm64
+FROM base-downloader AS base-downloader-linux-arm64
 ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu
+ENV DESCHASHPLUGIN_ARCH=linux-arm64
+ENV DESCHASHPLUGIN_HASH=d48c3e5aede77bd9cb72d78689ce12c0327f624435cb0496b3eacb92df416363
 
-FROM --platform=$BUILDPLATFORM base-downloader as base-downloader-linux-arm
+FROM base-downloader AS base-downloader-linux-arm
 ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf
+ENV DESCHASHPLUGIN_ARCH=linux-arm
+ENV DESCHASHPLUGIN_HASH=f7df336c72dd1674bd18ff23862a410b6a9691a3e13752264dcffa0950e21c74
 
-FROM base-downloader-${TARGETOS}-${TARGETARCH} as downloader
+FROM base-downloader-${TARGETOS}-${TARGETARCH} AS downloader
 
 RUN set -ex \
 	&& apt-get update \
@@ -29,7 +38,7 @@ RUN set -ex \
 WORKDIR /opt
 
 
-ARG BITCOIN_VERSION=22.0
+ENV BITCOIN_VERSION=27.1
 ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
 ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/$BITCOIN_TARBALL
 ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS
@@ -52,27 +61,36 @@ RUN mkdir /opt/litecoin && cd /opt/litecoin \
     && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \
     && rm litecoin.tar.gz
 
-FROM debian:bullseye-slim as builder
+ENV DESCHASHPLUGIN_URL https://github.com/nbd-wtf/invoicewithdescriptionhash/releases/download/v1.4/invoicewithdescriptionhash-v1.4-${DESCHASHPLUGIN_ARCH}.tar.gz
+ENV DESCHASHPLUGIN_SHA256 ${DESCHASHPLUGIN_HASH}
+RUN mkdir /opt/deschashplugin && cd /opt/deschashplugin \
+    && wget -qO invoicewithdescriptionhash.tar.gz "$DESCHASHPLUGIN_URL" \
+    && echo "$DESCHASHPLUGIN_SHA256  invoicewithdescriptionhash.tar.gz" | sha256sum -c - \
+    && tar -xzvf invoicewithdescriptionhash.tar.gz && rm invoicewithdescriptionhash.tar.gz \
+    && chmod a+x invoicewithdescriptionhash
 
-ENV LIGHTNINGD_VERSION=master
+FROM --platform=${DEFAULT_TARGETPLATFORM} ${BASE_DISTRO} AS base-builder
 RUN apt-get update -qq && \
     apt-get install -qq -y --no-install-recommends \
         autoconf \
         automake \
+        bison \
         build-essential \
         ca-certificates \
         curl \
         dirmngr \
+        flex \
         gettext \
         git \
         gnupg \
-        libpq-dev \
+        jq \
+        libicu-dev \
         libtool \
         libffi-dev \
         pkg-config \
         libssl-dev \
         protobuf-compiler \
-        python3.9 \
+        python3 \
         python3-dev \
         python3-mako \
         python3-pip \
@@ -82,80 +100,245 @@ RUN apt-get update -qq && \
         libevent-dev \
         qemu-user-static \
         wget \
-        jq
+        unzip \
+        tclsh
 
-RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz \
-    && tar xvf zlib-1.2.13.tar.gz \
-    && cd zlib-1.2.13 \
-    && ./configure \
-    && make \
-    && make install && cd .. && \
-    rm zlib-1.2.13.tar.gz && \
-    rm -rf zlib-1.2.13
+ENV PATH="/root/.local/bin:$PATH" \
+    PYTHON_VERSION=3 \
+    POETRY_VERSION=2.0.1
+RUN curl -sSL https://install.python-poetry.org | python3 - && \
+    poetry self add poetry-plugin-export
+RUN mkdir -p /root/.venvs && \
+    python3 -m venv /root/.venvs/cln && \
+    . /root/.venvs/cln/bin/activate && \
+    pip3 install --upgrade pip setuptools wheel
 
-RUN apt-get install -y --no-install-recommends unzip tclsh \
-    && wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip \
-    && unzip sqlite-src-3290000.zip \
-    && cd sqlite-src-3290000 \
-    && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension \
-    && make \
-    && make install && cd .. && rm sqlite-src-3290000.zip && rm -rf sqlite-src-3290000
-
-USER root
-ENV RUST_PROFILE=release
-ENV PATH=$PATH:/root/.cargo/bin/
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-RUN rustup toolchain install stable --component rustfmt --allow-downgrade
+RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz -O zlib.tar.gz && \
+    wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip -O sqlite.zip && \
+    wget -q https://ftp.postgresql.org/pub/source/v17.1/postgresql-17.1.tar.gz -O postgres.tar.gz
 
 WORKDIR /opt/lightningd
 COPY . /tmp/lightning
-
 RUN git clone --recursive /tmp/lightning . && \
     git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD)
 
+# Do not build python plugins (wss-proxy) here, python doesn't support cross compilation.
+RUN sed -i '/^wss-proxy/d' pyproject.toml && \
+    poetry lock && \
+    poetry export -o requirements.txt --without-hashes
+RUN mkdir -p /root/.venvs && \
+    python3 -m venv /root/.venvs/cln && \
+    . /root/.venvs/cln/bin/activate && \
+    pip3 install -r requirements.txt && \
+    pip3 cache purge
+WORKDIR /
+
+FROM base-builder AS base-builder-linux-amd64
+
+ENV POSTGRES_CONFIG="--without-readline" \
+    PG_CONFIG=/usr/local/pgsql/bin/pg_config
+
+FROM base-builder AS base-builder-linux-arm64
+ENV target_host=aarch64-linux-gnu \
+    target_host_rust=aarch64-unknown-linux-gnu \
+    target_host_qemu=qemu-aarch64-static
+
+RUN apt-get install -qq -y --no-install-recommends \
+        libc6-arm64-cross \
+        gcc-${target_host} \
+        g++-${target_host}
+
+ENV AR=${target_host}-ar \
+    AS=${target_host}-as \
+    CC=${target_host}-gcc \
+    CXX=${target_host}-g++ \
+    LD=${target_host}-ld \
+    STRIP=${target_host}-strip \
+    QEMU_LD_PREFIX=/usr/${target_host} \
+    HOST=${target_host} \
+    TARGET=${target_host_rust} \
+    RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
+    PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
+
+ENV ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
+    SQLITE_CONFIG="--host=${target_host} --prefix=${QEMU_LD_PREFIX}" \
+    POSTGRES_CONFIG="--without-readline --prefix=${QEMU_LD_PREFIX}" \
+    PG_CONFIG="${QEMU_LD_PREFIX}/bin/pg_config"
+
+FROM base-builder AS base-builder-linux-arm
+
+ENV target_host=arm-linux-gnueabihf \
+    target_host_rust=armv7-unknown-linux-gnueabihf \
+    target_host_qemu=qemu-arm-static
+
+RUN apt-get install -qq -y --no-install-recommends \
+        libc6-armhf-cross \
+        gcc-${target_host} \
+        g++-${target_host}
+
+ENV AR=${target_host}-ar \
+    AS=${target_host}-as \
+    CC=${target_host}-gcc \
+    CXX=${target_host}-g++ \
+    LD=${target_host}-ld \
+    STRIP=${target_host}-strip \
+    QEMU_LD_PREFIX=/usr/${target_host} \
+    HOST=${target_host} \
+    TARGET=${target_host_rust} \
+    RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
+    PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
+
+ENV ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
+    SQLITE_CONFIG="--host=${target_host} --prefix=${QEMU_LD_PREFIX}" \
+    POSTGRES_CONFIG="--without-readline --prefix=${QEMU_LD_PREFIX}" \
+    PG_CONFIG="${QEMU_LD_PREFIX}/bin/pg_config"
+
+FROM base-builder-${TARGETOS}-${TARGETARCH} AS builder
+
+ENV LIGHTNINGD_VERSION=master
+
+RUN mkdir zlib && tar xvf zlib.tar.gz -C zlib --strip-components=1 \
+    && cd zlib \
+    && ./configure ${ZLIB_CONFIG} \
+    && make \
+    && make install && cd .. && \
+    rm zlib.tar.gz && \
+    rm -rf zlib
+
+RUN unzip sqlite.zip \
+    && cd sqlite-* \
+    && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension ${SQLITE_CONFIG} \
+    && make \
+    && make install && cd .. && rm sqlite.zip && rm -rf sqlite-*
+
+RUN mkdir postgres && tar xvf postgres.tar.gz -C postgres --strip-components=1 \
+    && cd postgres \
+    && ./configure ${POSTGRES_CONFIG} \
+    && cd src/include \
+    && make install \
+    && cd ../interfaces/libpq \
+    && make install \
+    && cd ../../bin/pg_config \
+    && make install \
+    && cd ../../../../ && \
+    rm postgres.tar.gz && \
+    rm -rf postgres && \
+    ldconfig "$(${PG_CONFIG} --libdir)"
+
+# Save libpq to a specific location to copy it into the final image.
+RUN mkdir /var/libpq && cp -a "$(${PG_CONFIG} --libdir)"/libpq.* /var/libpq
+
+ENV RUST_PROFILE=release \
+    PATH="/root/.cargo/bin:/root/.local/bin:$PATH"
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
+RUN rustup toolchain install stable --component rustfmt --allow-downgrade
+
+COPY --from=downloader /usr/bin/${target_host_qemu} /usr/bin/${target_host_qemu}
+WORKDIR /opt/lightningd
+
+# If cross-compiling, need to tell it to cargo.
+RUN ( ! [ -n "${target_host}" ] ) || \
+    (mkdir -p .cargo && echo "[target.${target_host_rust}]\nlinker = \"${target_host}-gcc\"" > .cargo/config)
+
+# Weird errors with cargo for cln-grpc on arm7 https://github.com/ElementsProject/lightning/issues/6596
+RUN ( ! [ "${target_host}" = "arm-linux-gnueabihf" ] ) || \
+    (sed -i '/documentation = "https:\/\/docs.rs\/cln-grpc"/a include = ["**\/*.*"]' cln-grpc/Cargo.toml)
+
+# Ensure that the desired grpcio-tools & protobuf versions are installed
+# https://github.com/ElementsProject/lightning/pull/7376#issuecomment-2161102381
+RUN poetry lock && poetry install && \
+    poetry self add poetry-plugin-export
+
+# Ensure that git differences are removed before making bineries, to avoid `-modded` suffix
+# poetry.lock changed due to pyln-client, pyln-proto and pyln-testing version updates
+# pyproject.toml was updated to exclude wss-proxy plugins in base-builder stage
+RUN git reset --hard HEAD
+
+RUN ./configure --prefix=/tmp/lightning_install --enable-static && poetry run make install
+
+# Export the requirements for the plugins so we can install them in builder-python stage
+WORKDIR /opt/lightningd/plugins/wss-proxy
+RUN poetry lock && poetry export -o requirements.txt --without-hashes
+WORKDIR /opt/lightningd
+RUN echo 'RUSTUP_INSTALL_OPTS="${RUSTUP_INSTALL_OPTS}"' > /tmp/rustup_install_opts.txt
+
+# We need to build python plugins on the target's arch because python doesn't support cross build
+FROM ${BASE_DISTRO} AS builder-python
+RUN apt-get update -qq && \
+    apt-get install -qq -y --no-install-recommends \
+        git \
+        curl \
+        libtool \
+        pkg-config \
+        autoconf \
+        automake \
+        build-essential \
+        libffi-dev \
+        libssl-dev \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3-venv && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
 ENV PYTHON_VERSION=3
-RUN curl -sSL https://install.python-poetry.org | python3 -
+RUN mkdir -p /root/.venvs && \
+    python3 -m venv /root/.venvs/cln && \
+    . /root/.venvs/cln/bin/activate && \
+    pip3 install --upgrade pip setuptools wheel
 
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1
+# Copy rustup_install_opts.txt file from builder
+COPY --from=builder /tmp/rustup_install_opts.txt /tmp/rustup_install_opts.txt
+# Setup ENV $RUSTUP_INSTALL_OPTS for this stage
+RUN export $(cat /tmp/rustup_install_opts.txt)
+ENV PATH="/root/.cargo/bin:/root/.venvs/cln/bin:$PATH"
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
 
-RUN pip3 install --upgrade pip setuptools wheel
-RUN pip3 wheel cryptography
-RUN pip3 install grpcio-tools
-
-RUN /root/.local/bin/poetry export -o requirements.txt --without-hashes --with dev
+WORKDIR /opt/lightningd/plugins/wss-proxy
+COPY --from=builder /opt/lightningd/plugins/wss-proxy/requirements.txt .
 RUN pip3 install -r requirements.txt
+RUN pip3 cache purge
 
-RUN ./configure --prefix=/tmp/lightning_install --enable-static && \
-    make && \
-    /root/.local/bin/poetry run make install
+WORKDIR /opt/lightningd
 
-FROM debian:bullseye-slim as final
+FROM ${BASE_DISTRO} AS final
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
       tini \
       socat \
       inotify-tools \
-      python3.9 \
-      python3-pip \
-      qemu-user-static \
-      libpq5 && \
+      jq \
+      python3 \
+      python3-pip && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-ENV LIGHTNINGD_DATA=/root/.lightning
-ENV LIGHTNINGD_RPC_PORT=9835
-ENV LIGHTNINGD_PORT=9735
-ENV LIGHTNINGD_NETWORK=bitcoin
+ENV LIGHTNINGD_DATA=/root/.lightning \
+    LIGHTNINGD_RPC_PORT=9835 \
+    LIGHTNINGD_PORT=9735 \
+    LIGHTNINGD_NETWORK=bitcoin
 
 RUN mkdir $LIGHTNINGD_DATA && \
+    mkdir /etc/bundledplugins && \
+    mkdir $LIGHTNINGD_DATA/plugins && \
     touch $LIGHTNINGD_DATA/config
 VOLUME [ "/root/.lightning" ]
 
+# Take libpq directly from builder.
+RUN mkdir /var/libpq && mkdir -p /usr/local/pgsql/lib
+RUN --mount=type=bind,from=builder,source=/var/libpq,target=/var/libpq,rw \
+    cp -a /var/libpq/libpq.* /usr/local/pgsql/lib && \
+    echo "/usr/local/pgsql/lib" > /etc/ld.so.conf.d/libpq.conf && \
+    ldconfig
+
 COPY --from=builder /tmp/lightning_install/ /usr/local/
-COPY --from=builder /usr/local/lib/python3.9/dist-packages/ /usr/local/lib/python3.9/dist-packages/
+COPY --from=builder-python /root/.venvs/cln/lib/python3.11/site-packages /usr/local/lib/python3.11/dist-packages/
 COPY --from=downloader /opt/bitcoin/bin /usr/bin
 COPY --from=downloader /opt/litecoin/bin /usr/bin
+COPY --from=downloader /opt/deschashplugin $LIGHTNINGD_DATA/plugins
+COPY --from=downloader /opt/deschashplugin /etc/bundledplugins
 COPY tools/docker-entrypoint.sh entrypoint.sh
 
 EXPOSE 9735 9835

Makefile

@@ -1,10 +1,10 @@
 #! /usr/bin/make
 
 # Extract version from git, or if we're from a zipfile, use dirname
-VERSION=$(shell git describe --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|\1|gp')
+VERSION=$(shell git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|v\1|gp')
 
 # Next release.
-CLN_NEXT_VERSION := v24.05
+CLN_NEXT_VERSION := v25.05
 
 # --quiet / -s means quiet, dammit!
 ifeq ($(findstring s,$(word 1, $(MAKEFLAGS))),s)
@@ -26,7 +26,7 @@ CCANDIR := ccan
 
 # Where we keep the BOLT RFCs
 BOLTDIR := ../bolts/
-DEFAULT_BOLTVERSION := 6e85df448bfee7d10f26aabb06b8eba3d7505888
+DEFAULT_BOLTVERSION := ccfa38ed4f592c3711156bb4ded77f44ec01101d
 # Can be overridden on cmdline.
 BOLTVERSION := $(DEFAULT_BOLTVERSION)
 
@@ -36,7 +36,7 @@ SORT=LC_ALL=C sort
 
 
 ifeq ($V,1)
-VERBOSE = $(ECHO) '$(2)'; $(2)
+VERBOSE = $(ECHO) '$(subst ','\'',$(2))'; $(2)
 else
 VERBOSE = $(ECHO) $(1); $(2)
 endif
@@ -72,7 +72,7 @@ endif
 
 # (method=thread to support xdist)
 PYTEST_OPTS := -v -p no:logging $(PYTEST_OPTS)
-MY_CHECK_PYTHONPATH=$${PYTHONPATH}$${PYTHONPATH:+:}$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7
+MY_CHECK_PYTHONPATH=$${PYTHONPATH}$${PYTHONPATH:+:}$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7:$(shell pwd)/contrib/pyln-grpc-proto
 # Collect generated python files to be excluded from lint checks
 PYTHON_GENERATED= \
 	contrib/pyln-grpc-proto/pyln/grpc/primitives_pb2.py \
@@ -80,10 +80,6 @@ PYTHON_GENERATED= \
 	contrib/pyln-grpc-proto/pyln/grpc/node_pb2.py \
 	contrib/pyln-testing/pyln/testing/grpc2py.py
 
-# Options to pass to cppcheck. Mostly used to exclude files that are
-# generated with external tools that we don't have control over
-CPPCHECK_OPTS=-q --language=c --std=c11 --error-exitcode=1 --suppressions-list=.cppcheck-suppress --inline-suppr
-
 # This is where we add new features as bitcoin adds them.
 FEATURES :=
 
@@ -232,10 +228,24 @@ ALL_TEST_PROGRAMS :=
 ALL_TEST_GEN :=
 ALL_FUZZ_TARGETS :=
 ALL_C_SOURCES :=
-ALL_C_HEADERS := header_versions_gen.h version_gen.h
+ALL_C_HEADERS :=
 # Extra (non C) targets that should be built by default.
 DEFAULT_TARGETS :=
 
+# Installation directories
+exec_prefix = $(PREFIX)
+bindir = $(exec_prefix)/bin
+libexecdir = $(exec_prefix)/libexec
+pkglibexecdir = $(libexecdir)/$(PKGNAME)
+plugindir = $(pkglibexecdir)/plugins
+datadir = $(PREFIX)/share
+docdir = $(datadir)/doc/$(PKGNAME)
+mandir = $(datadir)/man
+man1dir = $(mandir)/man1
+man5dir = $(mandir)/man5
+man7dir = $(mandir)/man7
+man8dir = $(mandir)/man8
+
 # M1 macos machines with homebrew will install the native libraries in
 # /opt/homebrew instead of /usr/local, most likely because they
 # emulate x86_64 compatibility via Rosetta, and wanting to keep the
@@ -252,8 +262,8 @@ CPATH := /usr/local/include
 LIBRARY_PATH := /usr/local/lib
 endif
 
-CPPFLAGS += -DCLN_NEXT_VERSION="\"$(CLN_NEXT_VERSION)\"" -DBINTOPKGLIBEXECDIR="\"$(shell sh tools/rel.sh $(bindir) $(pkglibexecdir))\""
-CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I$(CPATH) $(SQLITE3_CFLAGS) $(POSTGRES_INCLUDE) $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) $(CSANFLAGS)
+CPPFLAGS += -DCLN_NEXT_VERSION="\"$(CLN_NEXT_VERSION)\"" -DPKGLIBEXECDIR="\"$(pkglibexecdir)\"" -DBINDIR="\"$(bindir)\"" -DPLUGINDIR="\"$(plugindir)\"" -DCCAN_TAL_NEVER_RETURN_NULL=1
+CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I$(CPATH) $(SQLITE3_CFLAGS) $(SODIUM_CFLAGS) $(POSTGRES_INCLUDE) $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) $(CSANFLAGS)
 
 # If CFLAGS is already set in the environment of make (to whatever value, it
 # does not matter) then it would export it to subprocesses with the above value
@@ -271,9 +281,9 @@ ifeq ($(STATIC),1)
 # For MacOS, Jacob Rapoport <jacob@rumblemonkey.com> changed this to:
 #  -L/usr/local/lib -lsqlite3 -lz -Wl,-lm -lpthread -ldl $(COVFLAGS)
 # But that doesn't static link.
-LDLIBS = -L$(CPATH) -Wl,-dn $(SQLITE3_LDLIBS) -lz -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
+LDLIBS = -L$(CPATH) -Wl,-dn $(SQLITE3_LDLIBS) -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
 else
-LDLIBS = -L$(CPATH) -lm $(SQLITE3_LDLIBS) -lz $(COVFLAGS)
+LDLIBS = -L$(CPATH) -lm $(SQLITE3_LDLIBS) $(COVFLAGS)
 endif
 
 # If we have the postgres client library we need to link against it as well
@@ -359,19 +369,23 @@ include doc/Makefile
 include contrib/msggen/Makefile
 include devtools/Makefile
 include tools/Makefile
+ifneq ($(RUST),0)
+include cln-rpc/Makefile
+include cln-grpc/Makefile
+endif
 include plugins/Makefile
 include tests/plugins/Makefile
 
 ifneq ($(FUZZING),0)
 	include tests/fuzz/Makefile
 endif
-ifneq ($(RUST),0)
-	include cln-rpc/Makefile
+
+ifneq ($V,1)
+MSGGEN_ARGS := -s
 endif
-include cln-grpc/Makefile
 
 $(MSGGEN_GENALL)&: contrib/msggen/msggen/schema.json
-	PYTHONPATH=contrib/msggen $(PYTHON) contrib/msggen/msggen/__main__.py generate
+	@$(call VERBOSE, "msggen $@", PYTHONPATH=contrib/msggen $(PYTHON) contrib/msggen/msggen/__main__.py $(MSGGEN_ARGS) generate)
 
 # The compiler assumes that the proto files are in the same
 # directory structure as the generated files will be. Since we
@@ -452,6 +466,11 @@ else
 PYTEST_OPTS += -x
 endif
 
+# Allow for targeting specific tests by setting the PYTEST_TESTS environment variable.
+ifeq ($(PYTEST_TESTS),)
+PYTEST_TESTS = "tests/"
+endif
+
 check-units:
 
 check: check-units installcheck pytest
@@ -462,7 +481,7 @@ ifeq ($(PYTEST),)
 	exit 1
 else
 # Explicitly hand VALGRIND so you can override on make cmd line.
-	PYTHONPATH=$(MY_CHECK_PYTHONPATH) TEST_DEBUG=1 VALGRIND=$(VALGRIND) $(PYTEST) tests/ $(PYTEST_OPTS)
+	PYTHONPATH=$(MY_CHECK_PYTHONPATH) TEST_DEBUG=1 VALGRIND=$(VALGRIND) $(PYTEST) $(PYTEST_TESTS) $(PYTEST_OPTS)
 endif
 
 check-fuzz: $(ALL_FUZZ_TARGETS)
@@ -540,13 +559,6 @@ check-pytest-pyln-proto:
 check-includes: check-src-includes check-hdr-includes
 	@tools/check-includes.sh
 
-# cppcheck gets confused by list_for_each(head, i, list): thinks i is uninit.
-.cppcheck-suppress:
-	@git ls-files -z -- "*.c" "*.h" | grep -vzE '^(ccan|contrib)/' | xargs -0 grep -n '_for_each' | sed 's/\([^:]*:.*\):.*/uninitvar:\1/' > $@
-
-check-cppcheck: .cppcheck-suppress
-	@trap 'rm -f .cppcheck-suppress' 0; git ls-files -z -- "*.c" "*.h" | grep -vzE '^ccan/' | xargs -0 cppcheck  ${CPPCHECK_OPTS}
-
 check-shellcheck:
 	@git ls-files -z -- "*.sh" | xargs -0 shellcheck -f gcc
 
@@ -563,12 +575,29 @@ check-discouraged-functions:
 # since it risks overflow.
 check-amount-access:
 	@! (git grep -nE "(->|\.)(milli)?satoshis" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" | grep -v '/* Raw:')
-	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*"
+	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" | grep -vE "sizeof.struct amount_(m)?sat."
+
+repeat-doc-examples:
+	@for i in $$(seq 1 $(n)); do \
+		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
+		echo "Iteration $$i" >> tests/autogenerate-examples-repeat.log; \
+		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
+		VALGRIND=0 TIMEOUT=40 TEST_DEBUG=1 GENERATE_EXAMPLES=1 pytest -vvv tests/autogenerate-rpc-examples.py; \
+		git diff >> tests/autogenerate-examples-repeat.log; \
+		git reset --hard; \
+		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
+	done
+
+update-doc-examples:
+	TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 $(PYTEST) $(PYTEST_OPTS) --timeout=1200 tests/autogenerate-rpc-examples.py && $(MAKE) $(MSGGEN_GEN_ALL)
+
+check-doc-examples: update-doc-examples
+	git diff --exit-code HEAD
 
 # For those without working cppcheck
 check-source-no-cppcheck: check-makefile check-source-bolt check-whitespace check-spelling check-python check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access
 
-check-source: check-source-no-cppcheck check-cppcheck
+check-source: check-source-no-cppcheck
 
 full-check: check check-source
 
@@ -636,7 +665,7 @@ version_gen.h: $(FORCE)
 endif
 
 # That forces this rule to be run every time, too.
-header_versions_gen.h: tools/headerversions
+header_versions_gen.h: tools/headerversions $(FORCE)
 	@tools/headerversions $@
 
 # We make a static library, this way linker can discard unused parts.
@@ -654,7 +683,7 @@ $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): %: %.o
 # uses some ccan modules internally).  We want to rely on -lwallycore etc.
 # (as per EXTERNAL_LDLIBS) so we filter them out here.
 $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS):
-	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a -o $@)
+	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a $($(@)_LDLIBS) -o $@)
 
 # We special case the fuzzing target binaries, as they need to link against libfuzzer,
 # which brings its own main().
@@ -685,6 +714,7 @@ update-ccan:
 
 # Now ALL_PROGRAMS is fully populated, we can expand it.
 all-programs: $(ALL_PROGRAMS)
+all-fuzz-programs: $(ALL_FUZZ_TARGETS)
 all-test-programs: $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS)
 default-targets: $(DEFAULT_TARGETS)
 
@@ -697,8 +727,10 @@ maintainer-clean: distclean
 	$(RM) $(PYTHON_GENERATED)
 
 # We used to have gen_ files, now we have _gen files.
+# We used to generate doc/schemas/lightning-sql.json.
 obsclean:
 	$(RM) gen_*.h */gen_*.[ch] */*/gen_*.[ch]
+	$(RM) doc/schemas/lightning-sql.json
 
 clean: obsclean
 	$(RM) libccan.a $(CCAN_OBJS) $(CDUMP_OBJS) $(ALL_OBJS)
@@ -717,7 +749,7 @@ clean: obsclean
 
 PYLNS=client proto testing
 # See doc/contribute-to-core-lightning/contributor-workflow.md
-update-py-versions: update-pyln-versions update-clnrest-version update-poetry-lock
+update-versions: update-pyln-versions update-wss-proxy-version update-poetry-lock update-dot-version update-doc-examples
 
 update-pyln-versions: $(PYLNS:%=update-pyln-version-%)
 
@@ -730,12 +762,20 @@ pyln-release:  $(PYLNS:%=pyln-release-%)
 pyln-release-%:
 	cd contrib/pyln-$* && $(MAKE) prod-release
 
-update-clnrest-version:
+update-wss-proxy-version:
 	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	cd plugins/clnrest && $(MAKE) upgrade-version
+	cd plugins/wss-proxy && $(MAKE) upgrade-version
 
 update-poetry-lock:
-	poetry update clnrest pyln-client pyln-proto pyln-testing
+	poetry update wss-proxy pyln-client pyln-proto pyln-testing update-reckless-version
+
+update-reckless-version:
+	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
+	@sed -i "s/__VERSION__ = '\([.-z]*\)'/__VERSION__ = '$(NEW_VERSION)'/" tools/reckless
+
+update-dot-version:
+	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
+	echo $(NEW_VERSION) > .version
 
 update-mocks: $(ALL_TEST_PROGRAMS:%=update-mocks/%.c)
 
@@ -744,22 +784,8 @@ $(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) lib
 update-mocks/%: % $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES)
 	@MAKE=$(MAKE) tools/update-mocks.sh "$*" $(SUPPRESS_OUTPUT)
 
-unittest/%: %
-	$(VG) $(VG_TEST_ARGS) $* > /dev/null
-
-# Installation directories
-exec_prefix = $(PREFIX)
-bindir = $(exec_prefix)/bin
-libexecdir = $(exec_prefix)/libexec
-pkglibexecdir = $(libexecdir)/$(PKGNAME)
-plugindir = $(pkglibexecdir)/plugins
-datadir = $(PREFIX)/share
-docdir = $(datadir)/doc/$(PKGNAME)
-mandir = $(datadir)/man
-man1dir = $(mandir)/man1
-man5dir = $(mandir)/man5
-man7dir = $(mandir)/man7
-man8dir = $(mandir)/man8
+unittest/%: % bolt-precheck
+	BOLTDIR=$(LOCAL_BOLTDIR) $(VG) $(VG_TEST_ARGS) $* > /dev/null
 
 # Commands
 MKDIR_P = mkdir -p
@@ -793,6 +819,7 @@ install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $
 	@$(NORMAL_INSTALL)
 	$(INSTALL_PROGRAM) $(BIN_PROGRAMS) $(DESTDIR)$(bindir)
 	$(INSTALL_PROGRAM) $(PKGLIBEXEC_PROGRAMS) $(DESTDIR)$(pkglibexecdir)
+	@if [ -d "$(DESTDIR)$(plugindir)/clnrest" ]; then rm -rf $(DESTDIR)$(plugindir)/clnrest; fi
 	[ -z "$(PLUGINS)" ] || $(INSTALL_PROGRAM) $(PLUGINS) $(DESTDIR)$(plugindir)
 	for PY in $(PY_PLUGINS); do DIR=`dirname $$PY`; DST=$(DESTDIR)$(plugindir)/`basename $$DIR`; if [ -d $$DST ]; then rm -rf $$DST; fi; $(INSTALL_PROGRAM) -d $$DIR; cp -a $$DIR $$DST ; done
 
@@ -819,6 +846,7 @@ install: install-program install-data
 TESTBINS = \
 	$(CLN_PLUGIN_EXAMPLES) \
 	tests/plugins/test_libplugin \
+	tests/plugins/channeld_fakenet \
 	tests/plugins/test_selfdisable_after_getmanifest \
 	tools/hsmtool
 
@@ -882,8 +910,11 @@ installcheck: all-programs
 	fi
 	@rm -rf testinstall || true
 
+version:
+	@echo ${VERSION}
+
 .PHONY: installdirs install-program install-data install uninstall \
-	installcheck ncc bin-tarball show-flags
+	installcheck ncc bin-tarball show-flags version
 
 # Make a tarball of opt/clightning/, optionally with label for distribution.
 ifneq ($(VERSION),)

README.md

@@ -19,25 +19,30 @@ Core Lightning (previously c-lightning) is a lightweight, highly customizable an
 
 ## Project Status
 
-[![Continuous Integration](https://github.com/ElementsProject/lightning/workflows/Continuous%20Integration/badge.svg)][actions]
-[![Pull Requests Welcome][prs]][prs-link]
-[![Irc][IRC]][IRC-link]
-[![Documentation Status](https://readthedocs.org/projects/lightning/badge/?version=docs)][docs]
+[![Continuous Integration][actions-badge]][actions]
+[![Pull Requests Welcome][prs-badge]][prs]
+[![Documentation Status][docs-badge]][docs]
+[![BoL2][bol2-badge]][bol2]
+[![Telegram][telegram-badge]][telegram]
+[![Discord][discord-badge]][discord]
+[![Irc][IRC-badge]][IRC]
 
 This implementation has been in production use on the Bitcoin mainnet since early 2018, with the launch of the [Blockstream Store][blockstream-store-blog].
-We recommend getting started by experimenting on `testnet` (or `regtest`), but the implementation is considered stable and can be safely used on mainnet.
+We recommend getting started by experimenting on `testnet` (`testnet4` or `regtest`), but the implementation is considered stable and can be safely used on mainnet.
+
+## Reach Out to Us
 
 Any help testing the implementation, reporting bugs, or helping with outstanding issues is very welcome.
-Don't hesitate to reach out to us on IRC at [#lightning-dev @ libera.chat][irc1], [#c-lightning @ libera.chat][irc2], or on the implementation-specific mailing list [c-lightning@lists.ozlabs.org][ml1], or on the Lightning Network-wide mailing list [lightning-dev@lists.linuxfoundation.org][ml2], or on Discord [core-lightning][discord], or on Telegram [Core Lightning][telegram].
+Don't hesitate to reach out to us on [Build-on-L2][bol2], or on the implementation-specific [mailing list][ml1], or on [CLN Discord][discord], or on [CLN Telegram][telegram], or on IRC at [dev][irc1]/[gen][irc2] channel.
 
 ## Getting Started
 
-Core Lightning only works on Linux and macOS, and requires a locally (or remotely) running `bitcoind` (version 22.0 or above) that is fully caught up with the network you're running on, and relays transactions (ie with `blocksonly=0`).
+Core Lightning only works on Linux and macOS, and requires a locally (or remotely) running `bitcoind` (version 25.0 or above) that is fully caught up with the network you're running on, and relays transactions (ie with `blocksonly=0`).
 Pruning (`prune=n` option in `bitcoin.conf`) is partially supported, see [here](#pruning) for more details.
 
 ### Installation
 
-There are 4 supported installation options:
+There are 3 supported installation options:
 
  - Installation of a pre-compiled binary from the [release page][releases] on GitHub.
  - Using one of the [provided docker images][dockerhub] on the Docker Hub.
@@ -101,13 +106,10 @@ the lightning network.
 There are also numerous plugins available for Core Lightning which add
 capabilities: in particular there's a collection at: https://github.com/lightningd/plugins
 
-Including [helpme][helpme-github] which guides you through setting up
-your first channels and customizing your node.
-
 For a less reckless experience, you can encrypt the HD wallet seed:
  see [HD wallet encryption](#hd-wallet-encryption).
 
-You can also chat to other users at [#c-lightning @ libera.chat][irc2];
+You can also chat to other users at Discord [core-lightning][discord];
 we are always happy to help you get started!
 
 
@@ -123,11 +125,11 @@ lightning-cli newaddr
 
 `lightningd` will register the funds once the transaction is confirmed.
 
-You may need to generate a p2sh-segwit address if the faucet does not support bech32:
+Alternatively you can generate a taproot address should your source of funds support it:
 
 ```bash
-# Return a p2sh-segwit address
-lightning-cli newaddr p2sh-segwit
+# Return a taproot address
+lightning-cli newaddr p2tr
 ```
 
 Confirm `lightningd` got funds by:
@@ -207,19 +209,24 @@ Developers wishing to contribute should start with the developer guide [here](do
 
 [blockstream-store-blog]: https://blockstream.com/2018/01/16/en-lightning-charge/
 [std]: https://github.com/lightning/bolts
-[prs]: https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat
-[prs-link]: http://makeapullrequest.com
-[IRC]: https://img.shields.io/badge/chat-on%20libera-brightgreen.svg
-[IRC-link]: https://web.libera.chat/#c-lightning
+[prs-badge]: https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat
+[prs]: http://makeapullrequest.com
+[bol2-badge]: https://badgen.net/badge/BoL2/chat/blue
+[bol2]: https://community.corelightning.org
+[ml1]: https://lists.ozlabs.org/listinfo/c-lightning
+[discord-badge]: https://badgen.net/badge/Discord/chat/blue
+[discord]: https://discord.gg/mE9s4rc5un
+[telegram-badge]: https://badgen.net/badge/Telegram/chat/blue
+[telegram]: https://t.me/lightningd
+[IRC-badge]: https://img.shields.io/badge/IRC-chat-blue.svg
+[IRC]: https://web.libera.chat/#c-lightning
 [irc1]: https://web.libera.chat/#lightning-dev
 [irc2]: https://web.libera.chat/#c-lightning
-[ml1]: https://lists.ozlabs.org/listinfo/c-lightning
-[ml2]: https://lists.linuxfoundation.org/mailman/listinfo/lightning-dev
-[discord]: https://discord.gg/mE9s4rc5un
-[telegram]: https://t.me/lightningd
+[docs-badge]: https://readthedocs.org/projects/lightning/badge/?version=docs
 [docs]: https://docs.corelightning.org/docs
 [releases]: https://github.com/ElementsProject/lightning/releases
 [dockerhub]: https://hub.docker.com/r/elementsproject/lightningd/
 [jsonrpcspec]: https://www.jsonrpc.org/specification
 [helpme-github]: https://github.com/lightningd/plugins/tree/master/helpme
+[actions-badge]: https://github.com/ElementsProject/lightning/workflows/Continuous%20Integration/badge.svg
 [actions]: https://github.com/ElementsProject/lightning/actions

SECURITY.md

@@ -6,19 +6,30 @@ We have a 3 month release cycle, and the last two versions are supported.
 
 ## Reporting a Vulnerability
 
-To report security issues send an email to rusty@rustcorp.com.au, or
-security@blockstream.com (not for support).
+To report security vulnerabilities, please send an email to one of the following addresses:
+- `rusty@rustcorp.com.au`
+- `security@blockstream.com`
+
+Note: These email addresses are exclusively for vulnerability reporting.
+
+For all other inquiries/communication, please refer to the [Reach Out to Us](https://github.com/ElementsProject/lightning?tab=readme-ov-file#reach-out-to-us) section in our README.
 
 ## Signatures For Releases
 
 The following keys may be used to communicate sensitive information to
 developers, and to validate signatures on releases:
 
-| Name | Fingerprint |
-|------|-------------|
-| Rusty Russell | 15EE 8D6C AB0E 7F0C F999  BFCB D920 0E6C D1AD B8F1 |
-| Christian Decker | B731 AAC5 21B0 1385 9313  F674 A26D 6D9F E088 ED58 |
-| Lisa Neigut | 30DE 693A E0DE 9E37 B3E7  EB6B BFF0 F678 10C1 EED1 |
-| Alex Myers | 0437 4E42 789B BBA9 462E  4767 F3BF 63F2 7474 36AB |
+| Name | Email | Fingerprint |
+|------|-------|-------------|
+| Blockstream Security Reporting | `security@blockstream.com` | 1176 542D A98E 71E1 3372  2EF7 4AC8 CC88 6844 A2D6 |
+| Rusty Russell | `rusty@rustcorp.com.au` | 15EE 8D6C AB0E 7F0C F999  BFCB D920 0E6C D1AD B8F1 |
+| Christian Decker | `decker@blockstream.com` | B731 AAC5 21B0 1385 9313  F674 A26D 6D9F E088 ED58 |
+| Lisa Neigut | `niftynei@gmail.com` | 30DE 693A E0DE 9E37 B3E7  EB6B BFF0 F678 10C1 EED1 |
+| Alex Myers | `alex@endothermic.dev` | 0437 4E42 789B BBA9 462E  4767 F3BF 63F2 7474 36AB |
+| Peter Neuroth | `pet.v.ne@gmail.com` | 653B 19F3 3DF7 EFF3 E9D1  C94C C3F2 1EE3 87FF 4CD2 |
+| Shahana Farooqui | `sfarooqui@blockstream.com` | FE13 58EB 7793 51DB 24E5  555A A327 573C 9758 9BF5 |
+| Blockstream CLN Release | `cln@blockstream.com` | 616C 52F9 9D06 12B2 A151  B107 4129 A994 AA7E 9852 |
 
-You can import a key by running the following command with that individual’s fingerprint: `gpg --keyserver hkps://keys.openpgp.org --recv-keys "<fingerprint>"` Ensure that you put quotes around fingerprints containing spaces.
+You can import a key by running the following command with that individual’s fingerprint: 
+`gpg --keyserver hkps://keys.openpgp.org --recv-keys "<fingerprint>"`. 
+Ensure that you put quotes around fingerprints containing spaces.

action.yml

@@ -1,6 +0,0 @@
----
-name: 'Lightning CI'
-description: 'A preconfigured container with all Core Lightning dependencies'
-runs:
-  using: 'docker'
-  image: 'contrib/Dockerfile.tester'

bitcoin/chainparams.c

@@ -137,6 +137,32 @@ const struct chainparams networks[] = {
      .bip32_key_version = {.bip32_pubkey_version = BIP32_VER_TEST_PUBLIC,
 			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
      .is_elements = false},
+    {.network_name = "testnet4",
+     .onchain_hrp = "tb",
+     .lightning_hrp = "tb",
+     .bip70_name = "testnet4",
+     // 00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043
+     .genesis_blockhash = {{{.u.u8 = {0x43, 0xf0, 0x8b, 0xda, 0xb0, 0x50, 0xe3,
+				      0x5b, 0x56, 0x7c, 0x86, 0x4b, 0x91, 0xf4,
+				      0x7f, 0x50, 0xae, 0x72, 0x5a, 0xe2, 0xde,
+				      0x53, 0xbc, 0xfb, 0xba, 0xf2, 0x84, 0xda,
+				      0x00, 0x00, 0x00, 0x00}}}},
+     .rpc_port = 48332,
+     .ln_port = 49735,
+     .cli = "bitcoin-cli",
+     .cli_args = "-testnet4",
+     .cli_min_supported_version = 150000,
+     .dust_limit = { 546 },
+     .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
+     .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
+     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
+     .p2pkh_version = 111,
+     .p2sh_version = 196,
+     .testnet = true,
+     .fee_asset_tag = NULL,
+     .bip32_key_version = {.bip32_pubkey_version = BIP32_VER_TEST_PUBLIC,
+			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
+     .is_elements = false},
     {.network_name = "litecoin",
      .onchain_hrp = "ltc",
      .lightning_hrp = "ltc",

bitcoin/psbt.c

@@ -102,6 +102,79 @@ struct wally_psbt *combine_psbt(const tal_t *ctx,
 	return combined_psbt;
 }
 
+static bool parent_or_grandparent(const tal_t *goal, const tal_t *child)
+{
+	const tal_t *parent = tal_parent(child);
+	if (!parent)
+		return false;
+	return parent == goal || parent_or_grandparent(goal, parent);
+}
+
+#define NULL_OR_MATCH_P(item, parent) \
+	((item) == NULL || parent_or_grandparent((parent), (item)))
+
+static void audit_map(const tal_t *ctx, const struct wally_map *map)
+{
+	assert(NULL_OR_MATCH_P(map->items, ctx));
+	for (size_t i = 0; i < map->num_items; i++) {
+		assert(NULL_OR_MATCH_P(map->items[i].key, ctx));
+		assert(NULL_OR_MATCH_P(map->items[i].value, ctx));
+		assert(!map->items[i].key
+			|| tal_bytelen(map->items[i].key)
+				== map->items[i].key_len);
+		assert(!map->items[i].value
+			|| tal_bytelen(map->items[i].value)
+				== map->items[i].value_len);
+	}
+}
+
+void audit_psbt(const tal_t *ctx, const struct wally_psbt *psbt)
+{
+	assert(psbt);
+	assert(NULL_OR_MATCH_P(psbt->tx, ctx));
+	assert(NULL_OR_MATCH_P(psbt->inputs, ctx));
+	assert(NULL_OR_MATCH_P(psbt->outputs, ctx));
+	audit_map(ctx, &psbt->unknowns);
+	audit_map(ctx, &psbt->global_xpubs);
+#ifndef WALLY_ABI_NO_ELEMENTS
+	audit_map(ctx, &psbt->global_scalars);
+#endif
+	for (size_t i = 0; i < psbt->num_inputs; i++) {
+		assert(NULL_OR_MATCH_P(psbt->inputs[i].utxo, ctx));
+		assert(NULL_OR_MATCH_P(psbt->inputs[i].witness_utxo, ctx));
+		assert(NULL_OR_MATCH_P(psbt->inputs[i].final_witness, ctx));
+		audit_map(ctx, &psbt->inputs[i].keypaths);
+		audit_map(ctx, &psbt->inputs[i].signatures);
+		audit_map(ctx, &psbt->inputs[i].unknowns);
+		audit_map(ctx, &psbt->inputs[i].preimages);
+		audit_map(ctx, &psbt->inputs[i].psbt_fields);
+		audit_map(ctx, &psbt->inputs[i].taproot_leaf_signatures);
+		audit_map(ctx, &psbt->inputs[i].taproot_leaf_scripts);
+		audit_map(ctx, &psbt->inputs[i].taproot_leaf_hashes);
+		audit_map(ctx, &psbt->inputs[i].taproot_leaf_paths);
+		/* DTODO: Investigate if taproot wally maps have child maps */
+#ifndef WALLY_ABI_NO_ELEMENTS
+		assert(NULL_OR_MATCH_P(psbt->inputs[i].pegin_tx, ctx));
+		assert(NULL_OR_MATCH_P(psbt->inputs[i].pegin_witness, ctx));
+		audit_map(ctx, &psbt->inputs[i].pset_fields);
+#endif
+	}
+	for (size_t i = 0; i < psbt->num_outputs; i++) {
+		assert(NULL_OR_MATCH_P(psbt->outputs[i].script, ctx));
+		assert(psbt->outputs[i].script_len
+			== tal_bytelen(psbt->outputs[i].script));
+		audit_map(ctx, &psbt->outputs[i].keypaths);
+		audit_map(ctx, &psbt->outputs[i].unknowns);
+		audit_map(ctx, &psbt->outputs[i].psbt_fields);
+		audit_map(ctx, &psbt->outputs[i].taproot_tree);
+		audit_map(ctx, &psbt->outputs[i].taproot_leaf_hashes);
+		audit_map(ctx, &psbt->outputs[i].taproot_leaf_paths);
+#ifndef WALLY_ABI_NO_ELEMENTS
+		audit_map(ctx, &psbt->outputs[i].pset_fields);
+#endif
+	}
+}
+
 bool psbt_is_finalized(const struct wally_psbt *psbt)
 {
 	size_t is_finalized;
@@ -311,6 +384,36 @@ bool psbt_input_have_signature(const struct wally_psbt *psbt,
 	return ok;
 }
 
+bool psbt_input_get_ecdsa_sig(const tal_t *ctx,
+			      const struct wally_psbt *psbt,
+			      size_t in,
+			      const struct pubkey *pubkey,
+			      struct bitcoin_signature **sig)
+{
+	u8 pk_der[PUBKEY_CMPR_LEN];
+	size_t index_plus_one;
+	struct wally_map_item *item;
+	bool ok;
+
+	assert(in < psbt->num_inputs);
+
+	pubkey_to_der(pk_der, pubkey);
+	*sig = NULL;
+
+	ok = wally_psbt_input_find_signature(&psbt->inputs[in], pk_der,
+					     sizeof(pk_der),
+					     &index_plus_one) == WALLY_OK;
+	if (ok) {
+		item = &psbt->inputs[in].signatures.items[index_plus_one - 1];
+		*sig = tal(ctx, struct bitcoin_signature);
+		if (!signature_from_der(item->value, item->value_len, *sig)) {
+			*sig = tal_free(*sig);
+			return false;
+		}
+	}
+	return ok;
+}
+
 void psbt_input_set_wit_utxo(struct wally_psbt *psbt, size_t in,
 			     const u8 *scriptPubkey, struct amount_sat amt)
 {
@@ -449,10 +552,9 @@ void psbt_elements_normalize_fees(struct wally_psbt *psbt)
 }
 
 void wally_psbt_input_get_txid(const struct wally_psbt_input *in,
-                 struct bitcoin_txid *txid)
+			       struct bitcoin_txid *txid)
 {
-    BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
-    memcpy(txid, in->txhash, sizeof(struct bitcoin_txid));
+	CROSS_TYPE_ASSIGNMENT(txid, &in->txhash);
 }
 
 bool psbt_has_input(const struct wally_psbt *psbt,
@@ -612,6 +714,17 @@ void *psbt_get_lightning(const struct wally_map *map,
 	return item->value;
 }
 
+void psbt_set_lightning(const tal_t *ctx,
+			struct wally_map *map,
+			const u8 proprietary_type,
+			const void *value,
+			size_t val_len)
+{
+	u8 *key = psbt_make_key(NULL, proprietary_type, NULL);
+	map_replace(ctx, map, key, value, val_len);
+	tal_free(key);
+}
+
 void psbt_output_set_unknown(const tal_t *ctx,
 			     struct wally_psbt_output *out,
 			     const u8 *key,
@@ -721,10 +834,9 @@ struct wally_psbt *psbt_from_b64(const tal_t *ctx,
 				 size_t b64len)
 {
 	struct wally_psbt *psbt;
-	char *str = tal_strndup(tmpctx, b64, b64len);
 
 	tal_wally_start();
-	if (wally_psbt_from_base64(str, /* flags */ 0, &psbt) == WALLY_OK)
+	if (wally_psbt_from_base64_n(b64, b64len, /* flags */ 0, &psbt) == WALLY_OK)
 		tal_add_destructor(psbt, psbt_destroy);
 	else
 		psbt = NULL;
@@ -886,25 +998,21 @@ struct amount_sat psbt_compute_fee(const struct wally_psbt *psbt)
 }
 
 bool wally_psbt_input_spends(const struct wally_psbt_input *input,
-               const struct bitcoin_outpoint *outpoint)
+			     const struct bitcoin_outpoint *outpoint)
 {
-    /* Useful, as tx_part can have some NULL inputs */
-    if (!input)
-        return false;
-    BUILD_ASSERT(sizeof(outpoint->txid) == sizeof(input->txhash));
+	/* Useful, as tx_part can have some NULL inputs */
+	if (!input)
+		return false;
 	if (input->index != outpoint->n)
 		return false;
-    if (memcmp(&outpoint->txid, input->txhash, sizeof(outpoint->txid)) != 0)
-        return false;
-    return true;
+	return CROSS_TYPE_EQ(&outpoint->txid, &input->txhash);
 }
 
 void wally_psbt_input_get_outpoint(const struct wally_psbt_input *in,
-                 struct bitcoin_outpoint *outpoint)
+				   struct bitcoin_outpoint *outpoint)
 {
-    BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
-    memcpy(&outpoint->txid, in->txhash, sizeof(struct bitcoin_txid));
-    outpoint->n = in->index;
+	CROSS_TYPE_ASSIGNMENT(&outpoint->txid, &in->txhash);
+	outpoint->n = in->index;
 }
 
 const u8 *wally_psbt_output_get_script(const tal_t *ctx,

bitcoin/psbt.h

@@ -61,6 +61,17 @@ struct wally_psbt *combine_psbt(const tal_t *ctx,
 				const struct wally_psbt *psbt0,
 				const struct wally_psbt *psbt1);
 
+/**
+ * audit_psbt - Audit the memory structure of the PSBT.
+ *
+ * This checks all known memory allocations in the PSBT and asserts that they
+ * are all allocated with 'ctx' being it's parent.
+ *
+ * ctx - the ctx all memory *should* be attached to
+ * psbt - the PSBT to audit.
+ * */
+void audit_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
+
 /**
  * psbt_is_finalized - Check if tx is ready to be extracted
  *
@@ -180,12 +191,20 @@ WARN_UNUSED_RESULT bool psbt_input_set_signature(struct wally_psbt *psbt, size_t
 
 /* Returns false on error. On success, *signature_found is set to true if the
  * input has a signature present for `pubkey` and false if if one was not found.
- * Only ignature presence is checked, is not validated. */
+ * Only signature presence is checked, it is not validated. */
 WARN_UNUSED_RESULT bool psbt_input_have_signature(const struct wally_psbt *psbt,
 						  size_t in,
 						  const struct pubkey *pubkey,
 						  bool *signature_found);
 
+/* Returns false on error. On success *sig is set to the signature otherwise
+ * *sig is set to NULL. */
+WARN_UNUSED_RESULT bool psbt_input_get_ecdsa_sig(const tal_t *ctx,
+						 const struct wally_psbt *psbt,
+						 size_t in,
+						 const struct pubkey *pubkey,
+						 struct bitcoin_signature **sig);
+
 void psbt_input_set_witscript(struct wally_psbt *psbt, size_t in, const u8 *wscript);
 
 /* psbt_input_set_unknown - Set the given Key-Value in the psbt's input keymap
@@ -212,6 +231,19 @@ void *psbt_get_lightning(const struct wally_map *map,
 			 const u8 proprietary_type,
 			 size_t *val_len);
 
+/* psbt_set_lightning - Set a propreitary lightning value on the given map
+ *
+ * @map - map of unknowns to set the value
+ * @proprietary_type - type no. to set
+ * @value - the value to be set
+ * @val_len - length of value
+ */
+void psbt_set_lightning(const tal_t *ctx,
+			struct wally_map *map,
+			const u8 proprietary_type,
+			const void *value,
+			size_t val_len);
+
 /* psbt_output_set_unknown - Set the given Key-Value in the psbt's output keymap
  *
  * @ctx - tal context for allocations

bitcoin/pubkey.c

@@ -83,7 +83,7 @@ char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key)
 
 int pubkey_cmp(const struct pubkey *a, const struct pubkey *b)
 {
-	u8 keya[33], keyb[33];
+	u8 keya[PUBKEY_CMPR_LEN], keyb[PUBKEY_CMPR_LEN];
 	pubkey_to_der(keya, a);
 	pubkey_to_der(keyb, b);
 	return memcmp(keya, keyb, sizeof(keya));

bitcoin/short_channel_id.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/compiler/compiler.h>
 #include <ccan/short_types/short_types.h>
-#include <ccan/structeq/structeq.h>
 #include <ccan/tal/tal.h>
 #include <common/gossip_constants.h>
 
@@ -19,6 +18,12 @@ static inline bool short_channel_id_eq(struct short_channel_id a,
 	return a.u64 == b.u64;
 }
 
+static inline size_t short_channel_id_hash(struct short_channel_id scid)
+{
+	/* scids cost money to generate, so simple hash works here */
+	return (scid.u64 >> 32) ^ (scid.u64 >> 16) ^ scid.u64;
+}
+
 /* BOLT #7:
  *
  * - MUST set `node_id_1` and `node_id_2` to the public keys of the two nodes
@@ -36,6 +41,12 @@ struct short_channel_id_dir {
 	int dir;
 };
 
+static inline bool short_channel_id_dir_eq(const struct short_channel_id_dir *a,
+					   const struct short_channel_id_dir *b)
+{
+	return short_channel_id_eq(a->scid, b->scid) && a->dir == b->dir;
+}
+
 static inline u32 short_channel_id_blocknum(struct short_channel_id scid)
 {
 	return scid.u64 >> 40;

bitcoin/test/run-psbt-from-tx.c

@@ -65,6 +65,9 @@ u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEED
 /* Generated stub for sha256_double */
 void sha256_double(struct sha256_double *shadouble UNNEEDED, const void *p UNNEEDED, size_t len UNNEEDED)
 { fprintf(stderr, "sha256_double called!\n"); abort(); }
+/* Generated stub for signature_from_der */
+bool signature_from_der(const u8 *der UNNEEDED, size_t len UNNEEDED, struct bitcoin_signature *sig UNNEEDED)
+{ fprintf(stderr, "signature_from_der called!\n"); abort(); }
 /* Generated stub for signature_to_der */
 size_t signature_to_der(u8 der[73] UNNEEDED, const struct bitcoin_signature *sig UNNEEDED)
 { fprintf(stderr, "signature_to_der called!\n"); abort(); }
@@ -111,6 +114,7 @@ int main(int argc, char *argv[])
 	const struct wally_map_item *final_scriptsig = wally_map_get_integer(&tx2->psbt->inputs[0].psbt_fields, /* PSBT_IN_FINAL_SCRIPTSIG */ 0x07);
 	assert(final_scriptsig->value_len > 0);
 	assert(tx2->psbt->inputs[0].final_witness != NULL);
+	audit_psbt(tx2->psbt, tx2->psbt);
 
 	common_shutdown();
 	return 0;

bitcoin/tx.c

@@ -403,8 +403,7 @@ void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
 	assert(innum < tx->wtx->num_inputs);
 
 	in = &tx->wtx->inputs[innum];
-	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
-	memcpy(in->txhash, &outpoint->txid, sizeof(struct bitcoin_txid));
+	CROSS_TYPE_ASSIGNMENT(&in->txhash, &outpoint->txid);
 	in->index = outpoint->n;
 }
 
@@ -412,15 +411,13 @@ void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
 void wally_tx_input_get_txid(const struct wally_tx_input *in,
 			     struct bitcoin_txid *txid)
 {
-	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
-	memcpy(txid, in->txhash, sizeof(struct bitcoin_txid));
+	CROSS_TYPE_ASSIGNMENT(txid, &in->txhash);
 }
 
 void wally_tx_input_get_outpoint(const struct wally_tx_input *in,
 				 struct bitcoin_outpoint *outpoint)
 {
-	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
-	memcpy(&outpoint->txid, in->txhash, sizeof(struct bitcoin_txid));
+	wally_tx_input_get_txid(in, &outpoint->txid);
 	outpoint->n = in->index;
 }
 
@@ -551,9 +548,13 @@ void bitcoin_tx_finalize(struct bitcoin_tx *tx)
 	assert(bitcoin_tx_check(tx));
 }
 
-struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt STEALS)
+struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt TAKES)
 {
 	size_t locktime;
+
+	if (!taken(psbt))
+		psbt = clone_psbt(tmpctx, psbt);
+
 	wally_psbt_get_locktime(psbt, &locktime);
 	struct bitcoin_tx *tx = bitcoin_tx(ctx, chainparams,
 					   psbt->num_inputs,
@@ -693,18 +694,6 @@ fail:
 	return NULL;
 }
 
-/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
-static void reverse_bytes(u8 *arr, size_t len)
-{
-	unsigned int i;
-
-	for (i = 0; i < len / 2; i++) {
-		unsigned char tmp = arr[i];
-		arr[i] = arr[len - 1 - i];
-		arr[len - 1 - i] = tmp;
-	}
-}
-
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid)
 {
@@ -824,8 +813,7 @@ bool wally_tx_input_spends(const struct wally_tx_input *input,
 	/* Useful, as tx_part can have some NULL inputs */
 	if (!input)
 		return false;
-	BUILD_ASSERT(sizeof(outpoint->txid) == sizeof(input->txhash));
-	if (memcmp(&outpoint->txid, input->txhash, sizeof(outpoint->txid)) != 0)
+	if (!CROSS_TYPE_EQ(&outpoint->txid, &input->txhash))
 		return false;
 	return input->index == outpoint->n;
 }
@@ -909,7 +897,8 @@ size_t bitcoin_tx_input_sig_weight(void)
 /* Input weight */
 size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight)
 {
-	size_t weight = witness_weight;
+	/* We assume < 253 witness elements */
+	size_t weight = 1 + witness_weight;
 
 	/* Input weight: txid + index + sequence */
 	weight += (32 + 4 + 4) * 4;
@@ -928,17 +917,32 @@ size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight)
 	return weight;
 }
 
-size_t bitcoin_tx_simple_input_witness_weight(void)
+size_t bitcoin_tx_input_witness_weight(enum utxotype utxotype)
 {
-	/* Account for witness (1 byte count + sig + key) */
-	return 1 + (bitcoin_tx_input_sig_weight() + 1 + 33);
-}
-
-/* We only do segwit inputs, and we assume witness is sig + key  */
-size_t bitcoin_tx_simple_input_weight(bool p2sh)
-{
-	return bitcoin_tx_input_weight(p2sh,
-				       bitcoin_tx_simple_input_witness_weight());
+	switch (utxotype) {
+	case UTXO_P2SH_P2WPKH:
+	case UTXO_P2WPKH:
+		/* Account for witness (sig + key) */
+		return bitcoin_tx_input_sig_weight() + 1 + 33;
+	case UTXO_P2WSH_FROM_CLOSE:
+		/* BOLT #3:
+		 * #### `to_remote` Output
+		 *
+		 * If `option_anchors` applies to the commitment
+		 * transaction, the `to_remote` output is encumbered by a one
+		 * block csv lock.
+		 *    <remotepubkey> OP_CHECKSIGVERIFY 1 OP_CHECKSEQUENCEVERIFY
+		 *
+		 * The output is spent by an input with `nSequence` field set
+		 * to `1` and witness: <remote_sig>
+		 * Otherwise, this output is a simple P2WPKH to `remotepubkey`.
+		 */
+		/* In practice, these predate anchors, so: */
+		return 1 + 1 + bitcoin_tx_input_sig_weight();
+	case UTXO_P2TR:
+		return 1 + 64;
+	}
+	abort();
 }
 
 size_t bitcoin_tx_2of2_input_witness_weight(void)

bitcoin/tx.h

@@ -48,6 +48,17 @@ struct bitcoin_tx_output {
 	u8 *script;
 };
 
+enum utxotype {
+	/* Obsolete: we used to have P2SH-wrapped outputs (removed in 24.02, though can still have old UTXOs) */
+	UTXO_P2SH_P2WPKH = 1,
+	/* "bech32" addresses */
+	UTXO_P2WPKH = 2,
+	/* Used for closing addresses: implies ->close_info is non-NULL */
+	UTXO_P2WSH_FROM_CLOSE = 3,
+	/* "p2tr" addresses. */
+	UTXO_P2TR = 4,
+};
+
 struct bitcoin_tx_output *new_tx_output(const tal_t *ctx,
 					struct amount_sat amount,
 					const u8 *script);
@@ -79,6 +90,18 @@ struct bitcoin_tx *clone_bitcoin_tx(const tal_t *ctx,
 struct bitcoin_tx *bitcoin_tx_from_hex(const tal_t *ctx, const char *hex,
 				       size_t hexlen);
 
+/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
+static inline void reverse_bytes(u8 *arr, size_t len)
+{
+	unsigned int i;
+
+	for (i = 0; i < len / 2; i++) {
+		unsigned char tmp = arr[i];
+		arr[i] = arr[len - 1 - i];
+		arr[len - 1 - i] = tmp;
+	}
+}
+
 /* Parse hex string to get txid (reversed, a-la bitcoind). */
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid);
@@ -88,7 +111,8 @@ bool bitcoin_txid_to_hex(const struct bitcoin_txid *txid,
 			 char *hexstr, size_t hexstr_len);
 
 /* Create a bitcoin_tx from a psbt */
-struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt);
+struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx,
+					struct wally_psbt *psbt TAKES);
 
 /* Internal de-linearization functions. */
 /* Pull a bitcoin tx, and create a PSBT wrapper for it */
@@ -303,14 +327,13 @@ size_t bitcoin_tx_output_weight(size_t outscript_len);
 /* Weight to push sig on stack. */
 size_t bitcoin_tx_input_sig_weight(void);
 
-/* Segwit input, but with parameter for witness weight (size) */
+/* Segwit input, but with parameter for witness weight (size).
+ * witness_weight must include the varint_size() for each witness element,
+ * but not the varint_size() for the number of elements. */
 size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight);
 
-/* The witness weight for a simple (sig + key) input */
-size_t bitcoin_tx_simple_input_witness_weight(void);
-
-/* We only do segwit inputs, and we assume witness is sig + key  */
-size_t bitcoin_tx_simple_input_weight(bool p2sh);
+/* The witness weight */
+size_t bitcoin_tx_input_witness_weight(enum utxotype utxotype);
 
 /* The witness for our 2of2 input (closing or commitment tx). */
 size_t bitcoin_tx_2of2_input_witness_weight(void);

ccan/README

@@ -1,3 +1,3 @@
 CCAN imported from http://ccodearchive.net.
 
-CCAN version: init-2578-g29e55f74
+CCAN version: init-2593-gca094039

ccan/ccan/bitmap/bitmap.c

@@ -9,7 +9,7 @@
 #define BIT_ALIGN_DOWN(n)	((n) & ~(BITMAP_WORD_BITS - 1))
 #define BIT_ALIGN_UP(n)		BIT_ALIGN_DOWN((n) + BITMAP_WORD_BITS - 1)
 
-void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m)
+void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
 	unsigned long am = BIT_ALIGN_DOWN(m);
@@ -19,22 +19,22 @@ void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m)
 	assert(m >= n);
 
 	if (am < an) {
-		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask & tailmask);
+		BITMAP_WORD(b, n) &= ~bitmap_bswap(headmask & tailmask);
 		return;
 	}
 
 	if (an > n)
-		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask);
+		BITMAP_WORD(b, n) &= ~bitmap_bswap(headmask);
 
 	if (am > an)
-		memset(&BITMAP_WORD(bitmap, an), 0,
+		memset(&BITMAP_WORD(b, an), 0,
 		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
 
 	if (m > am)
-		BITMAP_WORD(bitmap, m) &= ~bitmap_bswap(tailmask);
+		BITMAP_WORD(b, m) &= ~bitmap_bswap(tailmask);
 }
 
-void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m)
+void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
 	unsigned long am = BIT_ALIGN_DOWN(m);
@@ -44,19 +44,19 @@ void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m)
 	assert(m >= n);
 
 	if (am < an) {
-		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask & tailmask);
+		BITMAP_WORD(b, n) |= bitmap_bswap(headmask & tailmask);
 		return;
 	}
 
 	if (an > n)
-		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask);
+		BITMAP_WORD(b, n) |= bitmap_bswap(headmask);
 
 	if (am > an)
-		memset(&BITMAP_WORD(bitmap, an), 0xff,
+		memset(&BITMAP_WORD(b, an), 0xff,
 		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
 
 	if (m > am)
-		BITMAP_WORD(bitmap, m) |= bitmap_bswap(tailmask);
+		BITMAP_WORD(b, m) |= bitmap_bswap(tailmask);
 }
 
 static int bitmap_clz(bitmap_word w)
@@ -76,7 +76,7 @@ static int bitmap_clz(bitmap_word w)
 #endif
 }
 
-unsigned long bitmap_ffs(const bitmap *bitmap,
+unsigned long bitmap_ffs(const bitmap *b,
 			 unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
@@ -87,7 +87,7 @@ unsigned long bitmap_ffs(const bitmap *bitmap,
 	assert(m >= n);
 
 	if (am < an) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, n));
 
 		w &= (headmask & tailmask);
 
@@ -95,7 +95,7 @@ unsigned long bitmap_ffs(const bitmap *bitmap,
 	}
 
 	if (an > n) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, n));
 
 		w &= headmask;
 
@@ -104,7 +104,7 @@ unsigned long bitmap_ffs(const bitmap *bitmap,
 	}
 
 	while (an < am) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, an));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, an));
 
 		if (w)
 			return an + bitmap_clz(w);
@@ -113,7 +113,7 @@ unsigned long bitmap_ffs(const bitmap *bitmap,
 	}
 
 	if (m > am) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, m));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, m));
 
 		w &= tailmask;
 

ccan/ccan/bitmap/bitmap.h

@@ -58,37 +58,37 @@ static inline bitmap_word bitmap_bswap(bitmap_word w)
 #define BITMAP_TAIL(_bm, _nbits) \
 	(BITMAP_TAILWORD(_bm, _nbits) & BITMAP_TAILBITS(_nbits))
 
-static inline void bitmap_set_bit(bitmap *bitmap, unsigned long n)
+static inline void bitmap_set_bit(bitmap *b, unsigned long n)
 {
-	BITMAP_WORD(bitmap, n) |= BITMAP_WORDBIT(n);
+	BITMAP_WORD(b, n) |= BITMAP_WORDBIT(n);
 }
 
-static inline void bitmap_clear_bit(bitmap *bitmap, unsigned long n)
+static inline void bitmap_clear_bit(bitmap *b, unsigned long n)
 {
-	BITMAP_WORD(bitmap, n) &= ~BITMAP_WORDBIT(n);
+	BITMAP_WORD(b, n) &= ~BITMAP_WORDBIT(n);
 }
 
-static inline void bitmap_change_bit(bitmap *bitmap, unsigned long n)
+static inline void bitmap_change_bit(bitmap *b, unsigned long n)
 {
-	BITMAP_WORD(bitmap, n) ^= BITMAP_WORDBIT(n);
+	BITMAP_WORD(b, n) ^= BITMAP_WORDBIT(n);
 }
 
-static inline bool bitmap_test_bit(const bitmap *bitmap, unsigned long n)
+static inline bool bitmap_test_bit(const bitmap *b, unsigned long n)
 {
-	return !!(BITMAP_WORD(bitmap, n) & BITMAP_WORDBIT(n));
+	return !!(BITMAP_WORD(b, n) & BITMAP_WORDBIT(n));
 }
 
-void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m);
-void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m);
+void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m);
+void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m);
 
-static inline void bitmap_zero(bitmap *bitmap, unsigned long nbits)
+static inline void bitmap_zero(bitmap *b, unsigned long nbits)
 {
-	memset(bitmap, 0, bitmap_sizeof(nbits));
+	memset(b, 0, bitmap_sizeof(nbits));
 }
 
-static inline void bitmap_fill(bitmap *bitmap, unsigned long nbits)
+static inline void bitmap_fill(bitmap *b, unsigned long nbits)
 {
-	memset(bitmap, 0xff, bitmap_sizeof(nbits));
+	memset(b, 0xff, bitmap_sizeof(nbits));
 }
 
 static inline void bitmap_copy(bitmap *dst, const bitmap *src,
@@ -161,37 +161,36 @@ static inline bool bitmap_subset(const bitmap *src1, const bitmap *src2,
 	return true;
 }
 
-static inline bool bitmap_full(const bitmap *bitmap, unsigned long nbits)
+static inline bool bitmap_full(const bitmap *b, unsigned long nbits)
 {
 	unsigned long i;
 
 	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
-		if (bitmap[i].w != -1UL)
+		if (b[i].w != -1UL)
 			return false;
 	}
 	if (BITMAP_HASTAIL(nbits) &&
-	    (BITMAP_TAIL(bitmap, nbits) != BITMAP_TAILBITS(nbits)))
+	    (BITMAP_TAIL(b, nbits) != BITMAP_TAILBITS(nbits)))
 		return false;
 
 	return true;
 }
 
-static inline bool bitmap_empty(const bitmap *bitmap, unsigned long nbits)
+static inline bool bitmap_empty(const bitmap *b, unsigned long nbits)
 {
 	unsigned long i;
 
 	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
-		if (bitmap[i].w != 0)
+		if (b[i].w != 0)
 			return false;
 	}
-	if (BITMAP_HASTAIL(nbits) && (BITMAP_TAIL(bitmap, nbits) != 0))
+	if (BITMAP_HASTAIL(nbits) && (BITMAP_TAIL(b, nbits) != 0))
 		return false;
 
 	return true;
 }
 
-unsigned long bitmap_ffs(const bitmap *bitmap,
-			 unsigned long n, unsigned long m);
+unsigned long bitmap_ffs(const bitmap *b, unsigned long n, unsigned long m);
 
 /*
  * Allocation functions
@@ -221,26 +220,26 @@ static inline bitmap *bitmap_alloc1(unsigned long nbits)
 	return bitmap;
 }
 
-static inline bitmap *bitmap_realloc0(bitmap *bitmap,
+static inline bitmap *bitmap_realloc0(bitmap *b,
 				      unsigned long obits, unsigned long nbits)
 {
-	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
+	b = realloc(b, bitmap_sizeof(nbits));
 
-	if ((nbits > obits) && bitmap)
-		bitmap_zero_range(bitmap, obits, nbits);
+	if ((nbits > obits) && b)
+		bitmap_zero_range(b, obits, nbits);
 
-	return bitmap;
+	return b;
 }
 
-static inline bitmap *bitmap_realloc1(bitmap *bitmap,
+static inline bitmap *bitmap_realloc1(bitmap *b,
 				      unsigned long obits, unsigned long nbits)
 {
-	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
+	b = realloc(b, bitmap_sizeof(nbits));
 
-	if ((nbits > obits) && bitmap)
-		bitmap_fill_range(bitmap, obits, nbits);
+	if ((nbits > obits) && b)
+		bitmap_fill_range(b, obits, nbits);
 
-	return bitmap;
+	return b;
 }
 
 #endif /* CCAN_BITMAP_H_ */

ccan/ccan/crypto/siphash24/siphash24.h

@@ -80,7 +80,7 @@ void siphash24_init(struct siphash24_ctx *ctx, const struct siphash_seed *seed);
  * SIPHASH24_INIT - initializer for an SIPHASH24 context.
  * @seed1, @seed2: two 64-bit words for seed.
  *
- * This can be used to statically initialize an SIPHASH24 context (instead
+ * This can be used to staticly initialize an SIPHASH24 context (instead
  * of siphash24_init()).
  *
  * Example:

ccan/ccan/fdpass/fdpass.c

@@ -3,6 +3,7 @@
 #include <sys/socket.h>
 #include <errno.h>
 #include <string.h>
+#include <sys/types.h>
 
 bool fdpass_send(int sockout, int fd)
 {

ccan/ccan/htable/htable_type.h

@@ -1,19 +1,23 @@
 /* Licensed under LGPLv2+ - see LICENSE file for details */
 #ifndef CCAN_HTABLE_TYPE_H
 #define CCAN_HTABLE_TYPE_H
+#include "config.h"
+#include <assert.h>
 #include <ccan/htable/htable.h>
 #include <ccan/compiler/compiler.h>
-#include "config.h"
 
 /**
- * HTABLE_DEFINE_TYPE - create a set of htable ops for a type
+ * HTABLE_DEFINE_NODUPS_TYPE/HTABLE_DEFINE_DUPS_TYPE - create a set of htable ops for a type
  * @type: a type whose pointers will be values in the hash.
  * @keyof: a function/macro to extract a key: <keytype> @keyof(const type *elem)
  * @hashfn: a hash function for a @key: size_t @hashfn(const <keytype> *)
  * @eqfn: an equality function keys: bool @eqfn(const type *, const <keytype> *)
  * @prefix: a prefix for all the functions to define (of form <name>_*)
  *
- * NULL values may not be placed into the hash table.
+ * There are two variants, one of which allows duplicate keys, and one which
+ * does not.  The defined functions differ in some cases, as shown below.
+ *
+ * NULL values may not be placed into the hash table (nor (void *)1).
  *
  * This defines the type hashtable type and an iterator type:
  *	struct <name>;
@@ -33,15 +37,18 @@
  *
  * Delete and delete-by key return true if it was in the set:
  *	bool <name>_del(struct <name> *ht, const <type> *e);
- *	bool <name>_delkey(struct <name> *ht, const <keytype> *k);
+ *	bool <name>_delkey(struct <name> *ht, const <keytype> *k) (NODUPS only);
  *
  * Delete by iterator:
  *	bool <name>_delval(struct <name> *ht, struct <name>_iter *i);
  *
- * Find and return the (first) matching element, or NULL:
- *	type *<name>_get(const struct @name *ht, const <keytype> *k);
+ * Find and return the matching element, or NULL:
+ *	type *<name>_get(const struct @name *ht, const <keytype> *k) (NODUPS only);
  *
- * Find and return all matching elements, or NULL:
+ * Test for an element:
+ *	bool <name>_exists(const struct @name *ht, const <keytype> *k);
+ *
+ * Find and return all matching elements, or NULL (DUPS only):
  *	type *<name>_getfirst(const struct @name *ht, const <keytype> *k,
  *			      struct <name>_iter *i);
  *	type *<name>_getnext(const struct @name *ht, const <keytype> *k,
@@ -59,7 +66,7 @@
  * You can use HTABLE_INITIALIZER like so:
  *	struct <name> ht = { HTABLE_INITIALIZER(ht.raw, <name>_hash, NULL) };
  */
-#define HTABLE_DEFINE_TYPE(type, keyof, hashfn, eqfn, name)		\
+#define HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
 	struct name { struct htable raw; };				\
 	struct name##_iter { struct htable_iter i; };			\
 	static inline size_t name##_hash(const void *elem, void *priv)	\
@@ -89,66 +96,33 @@
 	{								\
 		return htable_copy(&dst->raw, &src->raw);		\
 	}								\
-	static inline bool name##_add(struct name *ht, const type *elem) \
-	{								\
-		return htable_add(&ht->raw, hashfn(keyof(elem)), elem);	\
-	}								\
 	static inline UNNEEDED bool name##_del(struct name *ht,		\
 					       const type *elem)	\
 	{								\
 		return htable_del(&ht->raw, hashfn(keyof(elem)), elem);	\
 	}								\
-	static inline UNNEEDED type *name##_get(const struct name *ht,	\
-				       const HTABLE_KTYPE(keyof, type) k) \
-	{								\
-		struct htable_iter i;					\
-		size_t h = hashfn(k);					\
-		void *c;						\
-									\
-		for (c = htable_firstval(&ht->raw,&i,h);		\
-		     c;							\
-		     c = htable_nextval(&ht->raw,&i,h)) {		\
-			if (eqfn(c, k))					\
-				return c;				\
-		}							\
-		return NULL;						\
-	}								\
 	static inline UNNEEDED type *name##_getmatch_(const struct name *ht, \
 				         const HTABLE_KTYPE(keyof, type) k, \
 				         size_t h,			\
 				         type *v,			\
-					 struct name##_iter *iter)	\
+					 struct htable_iter *iter)	\
 	{								\
 		while (v) {						\
 			if (eqfn(v, k))					\
 				break;					\
-			v = htable_nextval(&ht->raw, &iter->i, h);	\
+			v = htable_nextval(&ht->raw, iter, h);		\
 		}							\
 		return v;						\
 	}								\
-	static inline UNNEEDED type *name##_getfirst(const struct name *ht, \
-				         const HTABLE_KTYPE(keyof, type) k, \
-					 struct name##_iter *iter)	\
+	static inline UNNEEDED bool name##_exists(const struct name *ht,	\
+				       const HTABLE_KTYPE(keyof, type) k) \
 	{								\
+		struct htable_iter i;					\
 		size_t h = hashfn(k);					\
-		type *v = htable_firstval(&ht->raw, &iter->i, h);	\
-		return name##_getmatch_(ht, k, h, v, iter);			\
-	}								\
-	static inline UNNEEDED type *name##_getnext(const struct name *ht, \
-				         const HTABLE_KTYPE(keyof, type) k, \
-					 struct name##_iter *iter)	\
-	{								\
-		size_t h = hashfn(k);					\
-		type *v = htable_nextval(&ht->raw, &iter->i, h);	\
-		return name##_getmatch_(ht, k, h, v, iter);		\
-	}								\
-	static inline UNNEEDED bool name##_delkey(struct name *ht,	\
-					 const HTABLE_KTYPE(keyof, type) k) \
-	{								\
-		type *elem = name##_get(ht, k);				\
-		if (elem)						\
-			return name##_del(ht, elem);			\
-		return false;						\
+		void *v;						\
+									\
+		v = htable_firstval(&ht->raw, &i, h);			\
+		return name##_getmatch_(ht, k, h, v, &i) != NULL;	\
 	}								\
 	static inline UNNEEDED void name##_delval(struct name *ht,	\
 						  struct name##_iter *iter) \
@@ -177,6 +151,64 @@
 		return htable_prev(&ht->raw, &iter->i);			\
 	}
 
+#define HTABLE_DEFINE_NODUPS_TYPE(type, keyof, hashfn, eqfn, name)	\
+	HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
+	static inline UNNEEDED type *name##_get(const struct name *ht,	\
+				       const HTABLE_KTYPE(keyof, type) k) \
+	{								\
+		struct htable_iter i;					\
+		size_t h = hashfn(k);					\
+		void *v;						\
+									\
+		v = htable_firstval(&ht->raw, &i, h);			\
+		return name##_getmatch_(ht, k, h, v, &i);			\
+	}								\
+	static inline bool name##_add(struct name *ht, const type *elem) \
+	{								\
+		/* Open-coded for slightly more efficiency */		\
+		const HTABLE_KTYPE(keyof, type) k = keyof(elem);	\
+		struct htable_iter i;					\
+		size_t h = hashfn(k);					\
+		void *v;						\
+									\
+		v = htable_firstval(&ht->raw, &i, h);			\
+		assert(!name##_getmatch_(ht, k, h, v, &i));		\
+		return htable_add(&ht->raw, h, elem);			\
+	}								\
+	static inline UNNEEDED bool name##_delkey(struct name *ht,	\
+					 const HTABLE_KTYPE(keyof, type) k) \
+	{								\
+		type *elem = name##_get(ht, k);				\
+		if (elem)						\
+			return name##_del(ht, elem);			\
+		return false;						\
+	}
+
+#define HTABLE_DEFINE_DUPS_TYPE(type, keyof, hashfn, eqfn, name)	\
+	HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
+	static inline bool name##_add(struct name *ht, const type *elem) \
+	{								\
+		const HTABLE_KTYPE(keyof, type) k = keyof(elem);	\
+		return htable_add(&ht->raw, hashfn(k), elem);		\
+	}								\
+	static inline UNNEEDED type *name##_getfirst(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+					 struct name##_iter *iter)	\
+	{								\
+		size_t h = hashfn(k);					\
+		type *v = htable_firstval(&ht->raw, &iter->i, h);	\
+		return name##_getmatch_(ht, k, h, v, &iter->i);		\
+	}								\
+	static inline UNNEEDED type *name##_getnext(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+					 struct name##_iter *iter)	\
+	{								\
+		size_t h = hashfn(k);					\
+		type *v = htable_nextval(&ht->raw, &iter->i, h);	\
+		return name##_getmatch_(ht, k, h, v, &iter->i);		\
+	}
+
+
 #if HAVE_TYPEOF
 #define HTABLE_KTYPE(keyof, type) typeof(keyof((const type *)NULL))
 #else

ccan/ccan/htable/test/run-type-int.c

@@ -38,7 +38,7 @@ static bool cmp(const struct obj *obj, const unsigned int key)
 	return obj->key == key;
 }
 
-HTABLE_DEFINE_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
+HTABLE_DEFINE_NODUPS_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
 
 static void add_vals(struct htable_obj *ht,
 		     struct obj val[], unsigned int num)
@@ -112,14 +112,19 @@ static bool check_mask(struct htable *ht, const struct obj val[], unsigned num)
 	return true;
 }
 
+/* This variant allows duplicates! */
+HTABLE_DEFINE_DUPS_TYPE(struct obj, objkey, objhash, cmp, htable_obj_dups);
+
 int main(void)
 {
 	unsigned int i;
 	struct htable_obj ht, ht2;
+	struct htable_obj_dups ht_dups;
 	struct obj val[NUM_VALS], *result;
 	unsigned int dne;
 	void *p;
 	struct htable_obj_iter iter;
+	struct htable_obj_dups_iter dups_iter;
 
 	plan_tests(29);
 	for (i = 0; i < NUM_VALS; i++)
@@ -183,32 +188,35 @@ int main(void)
 	del_vals_bykey(&ht, val, NUM_VALS);
 	del_vals_bykey(&ht2, val, NUM_VALS);
 
+	/* Duplicates-allowed tests */
+	htable_obj_dups_init(&ht_dups);
 	/* Write two of the same value. */
 	val[1] = val[0];
-	htable_obj_add(&ht, &val[0]);
-	htable_obj_add(&ht, &val[1]);
+	htable_obj_dups_add(&ht_dups, &val[0]);
+	htable_obj_dups_add(&ht_dups, &val[1]);
 	i = 0;
 
-	result = htable_obj_getfirst(&ht, i, &iter);
+	result = htable_obj_dups_getfirst(&ht_dups, i, &dups_iter);
 	ok1(result == &val[0] || result == &val[1]);
 	if (result == &val[0]) {
-		ok1(htable_obj_getnext(&ht, i, &iter) == &val[1]);
-		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
+		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == &val[1]);
+		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_del(&ht, &val[0]));
-		ok1(htable_obj_getfirst(&ht, i, &iter) == &val[1]);
-		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
+		ok1(htable_obj_dups_del(&ht_dups, &val[0]));
+		ok1(htable_obj_dups_getfirst(&ht_dups, i, &dups_iter) == &val[1]);
+		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
 	} else {
-		ok1(htable_obj_getnext(&ht, i, &iter) == &val[0]);
-		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
+		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == &val[0]);
+		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_del(&ht, &val[1]));
-		ok1(htable_obj_getfirst(&ht, i, &iter) == &val[0]);
-		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
+		ok1(htable_obj_dups_del(&ht_dups, &val[1]));
+		ok1(htable_obj_dups_getfirst(&ht_dups, i, &dups_iter) == &val[0]);
+		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
 	}
 
+	htable_obj_dups_clear(&ht_dups);
 	htable_obj_clear(&ht);
 	htable_obj_clear(&ht2);
 	return exit_status();

ccan/ccan/htable/test/run-type.c

@@ -33,7 +33,10 @@ static bool cmp(const struct obj *obj, const unsigned int *key)
 	return obj->key == *key;
 }
 
-HTABLE_DEFINE_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
+HTABLE_DEFINE_NODUPS_TYPE(struct obj, objkey, objhash, cmp,
+			  htable_obj);
+HTABLE_DEFINE_DUPS_TYPE(struct obj, objkey, objhash, cmp,
+			htable_obj_dups);
 
 static void add_vals(struct htable_obj *ht,
 		     struct obj val[], unsigned int num)
@@ -111,12 +114,14 @@ int main(void)
 {
 	unsigned int i;
 	struct htable_obj ht, ht2;
+	struct htable_obj_dups ht_dups;
 	struct obj val[NUM_VALS], *result;
 	unsigned int dne;
 	void *p;
 	struct htable_obj_iter iter;
+	struct htable_obj_dups_iter dups_iter;
 
-	plan_tests(35);
+	plan_tests(36);
 	for (i = 0; i < NUM_VALS; i++)
 		val[i].key = i;
 	dne = i;
@@ -182,32 +187,37 @@ int main(void)
 	del_vals_bykey(&ht, val, NUM_VALS);
 	del_vals_bykey(&ht2, val, NUM_VALS);
 
+	/* Duplicates-allowed tests */
+	htable_obj_dups_init(&ht_dups);
+
 	/* Write two of the same value. */
 	val[1] = val[0];
-	htable_obj_add(&ht, &val[0]);
-	htable_obj_add(&ht, &val[1]);
+	htable_obj_dups_add(&ht_dups, &val[0]);
+	htable_obj_dups_add(&ht_dups, &val[1]);
 	i = 0;
 
-	result = htable_obj_getfirst(&ht, &i, &iter);
+	result = htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter);
 	ok1(result == &val[0] || result == &val[1]);
 	if (result == &val[0]) {
-		ok1(htable_obj_getnext(&ht, &i, &iter) == &val[1]);
-		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
+		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == &val[1]);
+		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_del(&ht, &val[0]));
-		ok1(htable_obj_getfirst(&ht, &i, &iter) == &val[1]);
-		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
+		ok1(htable_obj_dups_del(&ht_dups, &val[0]));
+		ok1(htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter) == &val[1]);
+		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
 	} else {
-		ok1(htable_obj_getnext(&ht, &i, &iter) == &val[0]);
-		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
+		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == &val[0]);
+		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_del(&ht, &val[1]));
-		ok1(htable_obj_getfirst(&ht, &i, &iter) == &val[0]);
-		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
+		ok1(htable_obj_dups_del(&ht_dups, &val[1]));
+		ok1(htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter) == &val[0]);
+		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
 	}
 
+	htable_obj_dups_clear(&ht_dups);
+	ok1(htable_obj_dups_count(&ht_dups) == 0);
 	htable_obj_clear(&ht);
 	ok1(htable_obj_count(&ht) == 0);
 	htable_obj_clear(&ht2);

ccan/ccan/htable/tools/density.c

@@ -26,7 +26,7 @@ static bool cmp(const ptrint_t *p, uintptr_t k)
 	return key(p) == k;
 }
 
-HTABLE_DEFINE_TYPE(ptrint_t, key, hash_uintptr, cmp, htable_ptrint);
+HTABLE_DEFINE_NODUPS_TYPE(ptrint_t, key, hash_uintptr, cmp, htable_ptrint);
 
 /* Nanoseconds per operation */
 static size_t normalize(const struct timeabs *start,

ccan/ccan/htable/tools/speed.c

@@ -33,7 +33,7 @@ static bool cmp(const struct object *object, const unsigned int *key)
 	return object->key == *key;
 }
 
-HTABLE_DEFINE_TYPE(struct object, objkey, hash_obj, cmp, htable_obj);
+HTABLE_DEFINE_NODUPS_TYPE(struct object, objkey, hash_obj, cmp, htable_obj);
 
 static unsigned int popcount(unsigned long val)
 {

ccan/ccan/htable/tools/stringspeed.c

@@ -31,7 +31,7 @@ static bool cmp(const char *obj, const char *key)
 	return strcmp(obj, key) == 0;
 }
 
-HTABLE_DEFINE_TYPE(char, strkey, hash_str, cmp, htable_str);
+HTABLE_DEFINE_NODUPS_TYPE(char, strkey, hash_str, cmp, htable_str);
 
 /* Nanoseconds per operation */
 static size_t normalize(const struct timeabs *start,

ccan/ccan/intmap/benchmark/speed.c

@@ -59,7 +59,7 @@ static bool eqfn(const struct htable_elem *elem, const uint64_t index)
 {
 	return elem->index == index;
 }
-HTABLE_DEFINE_TYPE(struct htable_elem, keyof, hashfn, eqfn, hash);
+HTABLE_DEFINE_NODUPS_TYPE(struct htable_elem, keyof, hashfn, eqfn, hash);
 
 static bool check_val(intmap_index_t i, uint64_t *v, uint64_t *expected)
 {

ccan/ccan/io/benchmarks/Makefile

@@ -1,16 +1,17 @@
-ALL:=run-loop run-different-speed run-length-prefix
+ALL:=run-loop run-different-speed run-length-prefix run-stream-many
 CCANDIR:=../../..
 CFLAGS:=-Wall -I$(CCANDIR) -O3 -flto
 LDFLAGS:=-O3 -flto
 LDLIBS:=-lrt
 
-OBJS:=time.o poll.o io.o err.o timer.o list.o
+OBJS:=time.o poll.o io.o err.o timer.o list.o ccan-tal.o ccan-take.o ccan-ilog.o
 
 default: $(ALL)
 
 run-loop: run-loop.o $(OBJS)
 run-different-speed: run-different-speed.o $(OBJS)
 run-length-prefix: run-length-prefix.o $(OBJS)
+run-stream-many: run-stream-many.o $(OBJS)
 
 time.o: $(CCANDIR)/ccan/time/time.c
 	$(CC) $(CFLAGS) -c -o $@ $<
@@ -24,6 +25,12 @@ io.o: $(CCANDIR)/ccan/io/io.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 err.o: $(CCANDIR)/ccan/err/err.c
 	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-ilog.o: $(CCANDIR)/ccan/ilog/ilog.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-tal.o: $(CCANDIR)/ccan/tal/tal.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-take.o: $(CCANDIR)/ccan/take/take.c
+	$(CC) $(CFLAGS) -c -o $@ $<
 
 clean:
 	rm -f *.o $(ALL)

ccan/ccan/io/benchmarks/run-stream-many.c

@@ -0,0 +1,132 @@
+/* Wait for many fds to connect, then try to stream the file to some of them in small chunks.
+ *
+ * This approximates the connectd behaviour in CLN, where we send gossip to peers.
+ */
+#include <ccan/io/io.h>
+#include <ccan/ptrint/ptrint.h>
+#include <ccan/time/time.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <err.h>
+#include <signal.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+
+/* We expect num_expected connections, and how many will be writers */
+static size_t max_readers, max_writers;
+
+/* How many raeaders and writers still going */
+static size_t num_readers, num_writers;
+
+/* How many times to do the write */
+static size_t write_iterations;
+
+/* The buffer to write */
+static char writebuf[256];
+
+/* We need this for readers, though we don't actually care! */
+static size_t len_ignored;
+
+struct timemono start_time;
+
+static void finished(void)
+{
+	struct timerel elapsed = timemono_since(start_time);
+	printf("Finished: %"PRIu64"usec\n", time_to_usec(elapsed));
+	exit(0);
+}
+
+static struct io_plan *write_loop(struct io_conn *conn, ptrint_t *iter)
+{
+	ptrdiff_t n = ptr2int(iter);
+
+	if (n > write_iterations) {
+		--num_writers;
+		if (num_writers == 0)
+			finished();
+		return io_wait(conn, conn, io_never, NULL);
+	}
+	return io_write(conn, writebuf, sizeof(writebuf), write_loop, int2ptr(n + 1));
+}
+
+static struct io_plan *read_loop(struct io_conn *conn, void *unused)
+{
+	return io_read_partial(conn, writebuf, sizeof(writebuf), &len_ignored, read_loop, unused);
+}
+
+static void reader_failed(struct io_conn *conn, intptr_t *num)
+{
+	err(1, "Reader %zu/%zu", (size_t)ptr2int(num), max_readers);
+}
+
+static void writer_failed(struct io_conn *conn, intptr_t *num)
+{
+	err(1, "Writer %zu/%zu", (size_t)ptr2int(num), max_writers);
+}
+
+static struct io_plan *connection_in(struct io_conn *conn, void *sleep_on)
+{
+	if (num_readers < max_readers) {
+		printf("r");
+		fflush(stdout);
+		num_readers++;
+		io_set_finish(conn, reader_failed, int2ptr(num_readers));
+		return read_loop(conn, NULL);
+	}
+
+	/* We assign writers last: not sure it matters, but it's more reflective
+	 * of lightning where more recent connections tend to ask for gossip */
+	num_writers++;
+	printf("w");
+	fflush(stdout);
+
+	io_set_finish(conn, writer_failed, int2ptr(num_writers));
+	io_set_finish(conn, writer_failed, NULL);
+	if (num_writers < max_writers)
+		return io_wait(conn, sleep_on, write_loop, int2ptr(0));
+
+	/* Everyone is connected.  Wake them and start final one */
+	io_wake(sleep_on);
+	printf("Starting!\n");
+	start_time = time_mono();
+	return write_loop(conn, int2ptr(0));
+}
+
+int main(int argc, char *argv[])
+{
+	int fd;
+	struct sockaddr_in s4;
+	int on = 1;
+
+	if (argc != 5)
+		errx(1, "Usage: <portnum> <num-idle> <num-streaming> <mb-streamed>");
+
+	memset(&s4, 0, sizeof(s4));
+	s4.sin_family = AF_INET;
+	s4.sin_port = htons(atol(argv[1]));
+	s4.sin_addr.s_addr = INADDR_ANY;
+
+	max_readers = atol(argv[2]);
+	max_writers = atol(argv[3]);
+	write_iterations = atol(argv[4]) * (1024 * 1024 / sizeof(writebuf));
+
+	fd = socket(AF_INET, SOCK_STREAM, 0);
+	if (fd < 0)
+		err(1, "Creating socket");
+
+	/* Re-use, please.. */
+	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)))
+		err(1, "Setting reuseaddr");
+
+	if (bind(fd, &s4, sizeof(s4)) != 0)
+		err(1, "Binding");
+
+	if (listen(fd, 1) != 0)
+		err(1, "Listening");
+
+	io_new_listener(NULL, fd, connection_in, &s4);
+	io_loop(NULL, NULL);
+	errx(1, "Sockets exited?");
+}

ccan/ccan/io/fdpass/fdpass.c

@@ -4,6 +4,12 @@
 #include <ccan/io/io_plan.h>
 #include <errno.h>
 
+static void destroy_conn_close_send_fd(struct io_conn *conn,
+				       struct io_plan_arg *arg)
+{
+	close(arg->u1.s);
+}
+
 static int do_fd_send(int fd, struct io_plan_arg *arg)
 {
 	if (!fdpass_send(fd, arg->u1.s)) {
@@ -12,8 +18,11 @@ static int do_fd_send(int fd, struct io_plan_arg *arg)
 			return 0;
 		return -1;
 	}
-	if (arg->u2.s)
+	if (arg->u2.vp) {
+		struct io_conn *conn = arg->u2.vp;
 		close(arg->u1.s);
+		tal_del_destructor2(conn, destroy_conn_close_send_fd, arg);
+	}
 	return 1;
 }
 
@@ -26,7 +35,11 @@ struct io_plan *io_send_fd_(struct io_conn *conn,
 	struct io_plan_arg *arg = io_plan_arg(conn, IO_OUT);
 
 	arg->u1.s = fd;
-	arg->u2.s = fdclose;
+	/* We need conn ptr for destructor */
+	arg->u2.vp = fdclose ? conn : NULL;
+	/* If conn closes before sending, we still need to close fd */
+	if (fdclose)
+		tal_add_destructor2(conn, destroy_conn_close_send_fd, arg);
 
 	return io_set_plan(conn, IO_OUT, do_fd_send, next, next_arg);
 }
@@ -39,7 +52,9 @@ static int do_fd_recv(int fd, struct io_plan_arg *arg)
 		/* In case ccan/io ever gets smart with non-blocking. */
 		if (errno == EAGAIN || errno == EWOULDBLOCK)
 			return 0;
-		return -1;
+		/* If they can't handle the error, this will close conn! */
+		if (!io_get_extended_errors())
+			return -1;
 	}
 	*(int *)arg->u1.vp = fdin;
 	return 1;

ccan/ccan/io/fdpass/fdpass.h

@@ -45,8 +45,9 @@ struct io_plan *io_send_fd_(struct io_conn *conn,
  * @arg: @next argument
  *
  * This creates a plan to receive a file descriptor, as sent by
- * io_send_fd.  Once it's all read, the @next function will be called:
- * on an error, the finish function is called instead.
+ * io_send_fd.  Once it's all read, the @next function will be called.
+ * On an error, if io_get_extended_errors() is true, then @next is called
+ * and @fd will be -1, otherwise the finish function is called.
  *
  * Note that the I/O may actually be done immediately.
  *

ccan/ccan/io/io.c

@@ -15,6 +15,7 @@
 void *io_loop_return;
 
 struct io_plan io_conn_freed;
+static bool io_extended_errors;
 
 struct io_listener *io_new_listener_(const tal_t *ctx, int fd,
 				     struct io_plan *(*init)(struct io_conn *,
@@ -383,9 +384,19 @@ void io_wake(const void *wait)
 	backend_wake(wait);
 }
 
-/* Returns false if this should not be touched (eg. freed). */
-static bool do_plan(struct io_conn *conn, struct io_plan *plan,
-		    bool idle_on_epipe)
+enum plan_result {
+	/* Destroyed, do not touch */
+	FREED,
+	/* Worked, call again. */
+	KEEP_GOING,
+	/* Failed with EAGAIN or did partial. */
+	EXHAUSTED,
+	/* No longer interested in read (or write) */
+	UNINTERESTED
+};
+
+static enum plan_result do_plan(struct io_conn *conn, struct io_plan *plan,
+				bool idle_on_epipe)
 {
 	/* We shouldn't have polled for this event if this wasn't true! */
 	assert(plan->status == IO_POLLING_NOTSTARTED
@@ -393,18 +404,26 @@ static bool do_plan(struct io_conn *conn, struct io_plan *plan,
 
 	switch (plan->io(conn->fd.fd, &plan->arg)) {
 	case -1:
+		/* This is expected, as we call optimistically! */
+		if (errno == EAGAIN)
+			return EXHAUSTED;
 		if (errno == EPIPE && idle_on_epipe) {
 			plan->status = IO_UNSET;
 			backend_new_plan(conn);
-			return false;
+			return UNINTERESTED;
 		}
 		io_close(conn);
-		return false;
+		return FREED;
 	case 0:
 		plan->status = IO_POLLING_STARTED;
-		return true;
+		/* If it started but didn't finish, don't call again. */
+		return EXHAUSTED;
 	case 1:
-		return next_plan(conn, plan);
+		if (!next_plan(conn, plan))
+			return FREED;
+		if (plan->status == IO_POLLING_NOTSTARTED)
+			return KEEP_GOING;
+		return UNINTERESTED;
 	default:
 		/* IO should only return -1, 0 or 1 */
 		abort();
@@ -413,16 +432,43 @@ static bool do_plan(struct io_conn *conn, struct io_plan *plan,
 
 void io_ready(struct io_conn *conn, int pollflags)
 {
-	if (pollflags & POLLIN)
-		if (!do_plan(conn, &conn->plan[IO_IN], false))
-			return;
+	enum plan_result res;
 
-	if (pollflags & POLLOUT)
-		/* If we're writing to a closed pipe, we need to wait for
-		 * read to fail if we're duplex: we want to drain it! */
-		do_plan(conn, &conn->plan[IO_OUT],
-			conn->plan[IO_IN].status == IO_POLLING_NOTSTARTED
-			|| conn->plan[IO_IN].status == IO_POLLING_STARTED);
+	if (pollflags & POLLIN) {
+		for (;;) {
+			res = do_plan(conn, &conn->plan[IO_IN], false);
+			switch (res) {
+			case FREED:
+				return;
+			case EXHAUSTED:
+			case UNINTERESTED:
+				goto try_write;
+			case KEEP_GOING:
+				continue;
+			}
+			abort();
+		}
+	}
+
+try_write:
+	if (pollflags & POLLOUT) {
+		for (;;) {
+			/* If we're writing to a closed pipe, we need to wait for
+			 * read to fail if we're duplex: we want to drain it! */
+			res = do_plan(conn, &conn->plan[IO_OUT],
+				      conn->plan[IO_IN].status == IO_POLLING_NOTSTARTED
+				      || conn->plan[IO_IN].status == IO_POLLING_STARTED);
+			switch (res) {
+			case FREED:
+			case EXHAUSTED:
+			case UNINTERESTED:
+				return;
+			case KEEP_GOING:
+				continue;
+			}
+			abort();
+		}
+	}
 }
 
 void io_do_always(struct io_plan *plan)
@@ -540,7 +586,7 @@ struct io_plan *io_sock_shutdown(struct io_conn *conn)
 	/* And leave unset .*/
 	return &conn->plan[IO_IN];
 }
-	
+
 bool io_flush_sync(struct io_conn *conn)
 {
 	struct io_plan *plan = &conn->plan[IO_OUT];
@@ -576,3 +622,13 @@ again:
 	io_fd_block(io_conn_fd(conn), false);
 	return ok;
 }
+
+void io_set_extended_errors(bool state)
+{
+	io_extended_errors = state;
+}
+
+bool io_get_extended_errors(void)
+{
+	return io_extended_errors;
+}

ccan/ccan/io/io.h

@@ -59,7 +59,7 @@ struct io_conn;
 	io_new_conn_((ctx), (fd),					\
 		     typesafe_cb_preargs(struct io_plan *, void *,	\
 					 (init), (arg),			\
-					 struct io_conn *conn),		\
+					 struct io_conn *),		\
 		     (void *)(arg))
 
 struct io_conn *io_new_conn_(const tal_t *ctx, int fd,
@@ -113,6 +113,8 @@ void io_set_finish_(struct io_conn *conn,
  * (tal'ocated off @ctx) and pass that to init().  Note that if there is
  * an error on this file descriptor, it will be freed.
  *
+ * Note: if the accept fails (usually due to EMFILE), init() will be called
+ * wth
  * Returns NULL on error (and sets errno).
  *
  * Example:
@@ -823,4 +825,13 @@ int (*io_poll_override(int (*poll)(struct pollfd *fds, nfds_t nfds, int timeout)
  */
 const void *io_have_fd(int fd, bool *listener);
 
+/**
+ * io_set_extended_errors - enable callbacks for errors.
+ * @state: true or false.
+ *
+ * Defaults false for compatibility.  See io_new_conn for what this changes.
+ */
+void io_set_extended_errors(bool state);
+bool io_get_extended_errors(void);
+
 #endif /* CCAN_IO_H */

ccan/ccan/io/poll.c

@@ -270,10 +270,12 @@ static void accept_conn(struct io_listener *l)
 {
 	int fd = accept(l->fd.fd, NULL, NULL);
 
-	/* FIXME: What to do here? */
-	if (fd < 0)
+	if (fd < 0) {
+		/* If they've enabled it, this is how we tell them */
+		if (io_get_extended_errors())
+			l->init(NULL, l->arg);
 		return;
-
+	}
 	io_new_conn(l->ctx, fd, l->init, l->arg);
 }
 

ccan/ccan/io/test/run-43-io_plan_in_started.c

@@ -18,9 +18,17 @@ static struct io_plan *init_in_conn(struct io_conn *conn, char *buf)
 	return io_read(conn, buf, 2, in_conn_done, NULL);
 }
 
+/* Every second time we say we're exhausted */
 static int do_nothing(int fd, struct io_plan_arg *arg)
 {
-	return 1;
+	static bool read_once;
+
+	read_once = !read_once;
+	if (read_once)
+		return 1;
+
+	errno = EAGAIN;
+	return -1;
 }
 
 static struct io_plan *dummy_write(struct io_conn *conn,

ccan/ccan/likely/likely.c

@@ -29,8 +29,8 @@ static bool trace_eq(const struct trace *t1, const struct trace *t2)
 }
 
 /* struct thash */
-HTABLE_DEFINE_TYPE(struct trace, (const struct trace *), hash_trace, trace_eq,
-		   thash);
+HTABLE_DEFINE_NODUPS_TYPE(struct trace, (const struct trace *), hash_trace, trace_eq,
+			  thash);
 
 static struct thash htable
 = { HTABLE_INITIALIZER(htable.raw, thash_hash, NULL) };

ccan/ccan/opt/opt.h

@@ -530,7 +530,7 @@ struct opt_table *opt_find_short(char arg);
  * You can set bits in type e.g. (1<<OPT_USER_START) to (1<<OPT_USER_END)
  * when calling _opt_register. */
 #define OPT_USER_START 8
-#define OPT_USER_END 15
+#define OPT_USER_END 30
 
 /* Below here are private declarations. */
 /* You can use this directly to build tables, but the macros will ensure
@@ -540,7 +540,7 @@ enum opt_type {
 	OPT_HASARG = 2,		/* -f arg|--foo=arg|--foo arg */
 	OPT_SUBTABLE = 4,	/* Actually, longopt points to a subtable... */
 	OPT_EARLY = 8,		/* Parse this from opt_early_parse() only. */
-	OPT_END = 16,		/* End of the table. */
+	OPT_END = 31,		/* End of the table. */
 
 	/* Make sure no compiler will assume we never have large
 	 * values in the enum! */

ccan/ccan/tal/str/str.c

@@ -14,21 +14,14 @@
 
 char *tal_strdup_(const tal_t *ctx, const char *p, const char *label)
 {
-	/* We have to let through NULL for take(). */
-	return tal_dup_arr_label(ctx, char, p, p ? strlen(p) + 1: 1, 0, label);
+	return tal_dup_arr_label(ctx, char, p, strlen(p) + 1, 0, label);
 }
 
 char *tal_strndup_(const tal_t *ctx, const char *p, size_t n, const char *label)
 {
-	size_t len;
+	size_t len = strnlen(p, n);
 	char *ret;
 
-	/* We have to let through NULL for take(). */
-	if (likely(p))
-		len = strnlen(p, n);
-	else
-		len = n;
-
 	ret = tal_dup_arr_label(ctx, char, p, len, 1, label);
 	if (ret)
 		ret[len] = '\0';
@@ -84,9 +77,6 @@ char *tal_vfmt_(const tal_t *ctx, const char *fmt, va_list ap, const char *label
 {
 	char *buf;
 
-	if (!fmt && taken(fmt))
-		return NULL;
-
 	/* A decent guess to start. */
 	buf = tal_arr_label(ctx, char, strlen(fmt) * 2, label);
 	if (!do_vfmt(&buf, 0, fmt, ap))
@@ -96,9 +86,6 @@ char *tal_vfmt_(const tal_t *ctx, const char *fmt, va_list ap, const char *label
 
 bool tal_append_vfmt(char **baseptr, const char *fmt, va_list ap)
 {
-	if (!fmt && taken(fmt))
-		return false;
-
 	return do_vfmt(baseptr, strlen(*baseptr), fmt, ap);
 }
 
@@ -120,13 +107,7 @@ char *tal_strcat_(const tal_t *ctx, const char *s1, const char *s2,
 	size_t len1, len2;
 	char *ret;
 
-	if (unlikely(!s2) && taken(s2)) {
-		if (taken(s1))
-			tal_free(s1);
-		return NULL;
-	}
-	/* We have to let through NULL for take(). */
-	len1 = s1 ? strlen(s1) : 0;
+	len1 = strlen(s1);
 	len2 = strlen(s2);
 
 	ret = tal_dup_arr_label(ctx, char, s1, len1, len2 + 1, label);
@@ -151,13 +132,11 @@ char **tal_strsplit_(const tal_t *ctx,
 			tal_free(string);
 		if (taken(delims))
 			tal_free(delims);
-		return NULL;
+		return parts;
 	}
 	str = tal_strdup(parts, string);
 	if (unlikely(!str))
 		goto fail;
-	if (unlikely(!delims) && is_taken(delims))
-		goto fail;
 
 	if (flags == STR_NO_EMPTY)
 		str += strspn(str, delims);
@@ -185,10 +164,14 @@ char **tal_strsplit_(const tal_t *ctx,
 	return parts;
 
 fail:
+#ifdef CCAN_TAL_NEVER_RETURN_NULL
+	abort();
+#else
 	tal_free(parts);
 	if (taken(delims))
 		tal_free(delims);
 	return NULL;
+#endif
 }
 
 char *tal_strjoin_(const tal_t *ctx,
@@ -199,12 +182,6 @@ char *tal_strjoin_(const tal_t *ctx,
 	char *ret = NULL;
 	size_t totlen = 0, dlen;
 
-	if (unlikely(!strings) && is_taken(strings))
-		goto fail;
-
-	if (unlikely(!delim) && is_taken(delim))
-		goto fail;
-
 	dlen = strlen(delim);
 	ret = tal_arr_label(ctx, char, dlen*2+1, label);
 	if (!ret)
@@ -269,15 +246,9 @@ bool tal_strreg_(const tal_t *ctx, const char *string, const char *label,
 	unsigned int i;
 	va_list ap;
 
-	if (unlikely(!regex) && is_taken(regex))
-		goto fail_no_re;
-
 	if (regcomp(&r, regex, REG_EXTENDED) != 0)
 		goto fail_no_re;
 
-	if (unlikely(!string) && is_taken(string))
-		goto fail;
-
 	if (regexec(&r, string, nmatch, matches, 0) != 0)
 		goto fail;
 

ccan/ccan/tal/str/str.h

@@ -12,17 +12,18 @@
 /**
  * tal_strdup - duplicate a string
  * @ctx: NULL, or tal allocated object to be parent.
- * @p: the string to copy (can be take()).
+ * @p: the string to copy (can be take(), must not be NULL).
  *
  * The returned string will have tal_count() == strlen() + 1.
  */
 #define tal_strdup(ctx, p) tal_strdup_(ctx, p, TAL_LABEL(char, "[]"))
-char *tal_strdup_(const tal_t *ctx, const char *p TAKES, const char *label);
+char *tal_strdup_(const tal_t *ctx, const char *p TAKES, const char *label)
+	TAL_RETURN_PTR NON_NULL_ARGS(2);
 
 /**
  * tal_strndup - duplicate a limited amount of a string.
  * @ctx: NULL, or tal allocated object to be parent.
- * @p: the string to copy (can be take()).
+ * @p: the string to copy (can be take(), must not be NULL).
  * @n: the maximum length to copy.
  *
  * Always gives a nul-terminated string, with strlen() <= @n.
@@ -30,24 +31,25 @@ char *tal_strdup_(const tal_t *ctx, const char *p TAKES, const char *label);
  */
 #define tal_strndup(ctx, p, n) tal_strndup_(ctx, p, n, TAL_LABEL(char, "[]"))
 char *tal_strndup_(const tal_t *ctx, const char *p TAKES, size_t n,
-		   const char *label);
+		   const char *label)
+	TAL_RETURN_PTR NON_NULL_ARGS(2);
 
 /**
  * tal_fmt - allocate a formatted string
  * @ctx: NULL, or tal allocated object to be parent.
- * @fmt: the printf-style format (can be take()).
+ * @fmt: the printf-style format (can be take(), must not be NULL).
  *
  * The returned string will have tal_count() == strlen() + 1.
  */
 #define tal_fmt(ctx, ...)				 \
 	tal_fmt_(ctx, TAL_LABEL(char, "[]"), __VA_ARGS__)
 char *tal_fmt_(const tal_t *ctx, const char *label, const char *fmt TAKES,
-	        ...) PRINTF_FMT(3,4);
+	       ...) PRINTF_FMT(3,4) TAL_RETURN_PTR NON_NULL_ARGS(3);
 
 /**
  * tal_vfmt - allocate a formatted string (va_list version)
  * @ctx: NULL, or tal allocated object to be parent.
- * @fmt: the printf-style format (can be take()).
+ * @fmt: the printf-style format (can be take(), must not be NULL).
  * @va: the va_list containing the format args.
  *
  * The returned string will have tal_count() == strlen() + 1.
@@ -56,40 +58,42 @@ char *tal_fmt_(const tal_t *ctx, const char *label, const char *fmt TAKES,
 	tal_vfmt_(ctx, fmt, va, TAL_LABEL(char, "[]"))
 char *tal_vfmt_(const tal_t *ctx, const char *fmt TAKES, va_list ap,
 		const char *label)
-	PRINTF_FMT(2,0);
+	PRINTF_FMT(2,0) TAL_RETURN_PTR NON_NULL_ARGS(2);
 
 /**
  * tal_append_fmt - append a formatted string to a talloc string.
  * @baseptr: a pointer to the tal string to be appended to.
- * @fmt: the printf-style format (can be take()).
+ * @fmt: the printf-style format (can be take(), must not be NULL).
  *
  * Returns false on allocation failure.
  * Otherwise tal_count(*@baseptr) == strlen(*@baseptr) + 1.
  */
-bool tal_append_fmt(char **baseptr, const char *fmt TAKES, ...) PRINTF_FMT(2,3);
+bool tal_append_fmt(char **baseptr, const char *fmt TAKES, ...)
+	PRINTF_FMT(2,3) NON_NULL_ARGS(2);
 
 /**
  * tal_append_vfmt - append a formatted string to a talloc string (va_list)
  * @baseptr: a pointer to the tal string to be appended to.
- * @fmt: the printf-style format (can be take()).
+ * @fmt: the printf-style format (can be take(), must not be NULL).
  * @va: the va_list containing the format args.
  *
  * Returns false on allocation failure.
  * Otherwise tal_count(*@baseptr) == strlen(*@baseptr) + 1.
  */
-bool tal_append_vfmt(char **baseptr, const char *fmt TAKES, va_list ap);
+bool tal_append_vfmt(char **baseptr, const char *fmt TAKES, va_list ap)
+	NON_NULL_ARGS(2);
 
 /**
  * tal_strcat - join two strings together
  * @ctx: NULL, or tal allocated object to be parent.
- * @s1: the first string (can be take()).
- * @s2: the second string (can be take()).
+ * @s1: the first string (can be take(), must not be NULL).
+ * @s2: the second string (can be take(), must not be NULL).
  *
  * The returned string will have tal_count() == strlen() + 1.
  */
 #define tal_strcat(ctx, s1, s2) tal_strcat_(ctx, s1, s2, TAL_LABEL(char, "[]"))
 char *tal_strcat_(const tal_t *ctx, const char *s1 TAKES, const char *s2 TAKES,
-		  const char *label);
+		  const char *label) TAL_RETURN_PTR NON_NULL_ARGS(2,3);
 
 enum strsplit {
 	STR_EMPTY_OK,
@@ -99,8 +103,8 @@ enum strsplit {
 /**
  * tal_strsplit - Split string into an array of substrings
  * @ctx: the context to tal from (often NULL).
- * @string: the string to split (can be take()).
- * @delims: delimiters where lines should be split (can be take()).
+ * @string: the string to split (can be take(), must not be NULL).
+ * @delims: delimiters where lines should be split (can be take(), must not be NULL).
  * @flags: whether to include empty substrings.
  *
  * This function splits a single string into multiple strings.
@@ -137,7 +141,8 @@ char **tal_strsplit_(const tal_t *ctx,
 		     const char *string TAKES,
 		     const char *delims TAKES,
 		     enum strsplit flag,
-		     const char *label);
+		     const char *label)
+	TAL_RETURN_PTR NON_NULL_ARGS(2,3);
 
 enum strjoin {
 	STR_TRAIL,
@@ -147,8 +152,8 @@ enum strjoin {
 /**
  * tal_strjoin - Join an array of substrings into one long string
  * @ctx: the context to tal from (often NULL).
- * @strings: the NULL-terminated array of strings to join (can be take())
- * @delim: the delimiter to insert between the strings (can be take())
+ * @strings: the NULL-terminated array of strings to join (can be take(), must not be NULL)
+ * @delim: the delimiter to insert between the strings (can be take(), must not be NULL)
  * @flags: whether to add a delimieter to the end
  *
  * This function joins an array of strings into a single string.  The
@@ -175,13 +180,14 @@ char *tal_strjoin_(const void *ctx,
 		   char *strings[] TAKES,
 		   const char *delim TAKES,
 		   enum strjoin flags,
-		   const char *label);
+		   const char *label)
+	TAL_RETURN_PTR NON_NULL_ARGS(2,3);
 
 /**
  * tal_strreg - match/extract from a string via (extended) regular expressions.
  * @ctx: the context to tal from (often NULL)
- * @string: the string to try to match (can be take())
- * @regex: the regular expression to match (can be take())
+ * @string: the string to try to match (can be take(), must not be NULL)
+ * @regex: the regular expression to match (can be take(), must not be NULL)
  * ...: pointers to strings to allocate for subexpressions.
  *
  * Returns true if we matched, in which case any parenthesized
@@ -221,5 +227,6 @@ char *tal_strjoin_(const void *ctx,
 #define tal_strreg(ctx, string, ...)					\
 	tal_strreg_(ctx, string, TAL_LABEL(char, "[]"), __VA_ARGS__)
 bool tal_strreg_(const void *ctx, const char *string TAKES,
-		 const char *label, const char *regex, ...);
+		 const char *label, const char *regex TAKES, ...)
+	NON_NULL_ARGS(2,4);
 #endif /* CCAN_STR_TAL_H */

ccan/ccan/tal/tal.c

@@ -456,13 +456,24 @@ static void del_tree(struct tal_hdr *t, const tal_t *orig, int saved_errno)
         freefn(t);
 }
 
+/* Don't have compiler complain we're returning NULL if we promised not to! */
+static void *null_alloc_failed(void)
+{
+#ifdef CCAN_TAL_NEVER_RETURN_NULL
+	abort();
+#else
+	return NULL;
+#endif /* CCAN_TAL_NEVER_RETURN_NULL */
+}
+
 void *tal_alloc_(const tal_t *ctx, size_t size, bool clear, const char *label)
 {
         struct tal_hdr *child, *parent = debug_tal(to_tal_hdr_or_null(ctx));
 
         child = allocate(sizeof(struct tal_hdr) + size);
 	if (!child)
-		return NULL;
+		return null_alloc_failed();
+
 	if (clear)
 		memset(from_tal_hdr(child), 0, size);
         child->prop = (void *)label;
@@ -470,7 +481,7 @@ void *tal_alloc_(const tal_t *ctx, size_t size, bool clear, const char *label)
 
         if (!add_child(parent, child)) {
 		freefn(child);
-		return NULL;
+		return null_alloc_failed();
 	}
 	debug_tal(parent);
 	if (notifiers)
@@ -501,7 +512,7 @@ void *tal_alloc_arr_(const tal_t *ctx, size_t size, size_t count, bool clear,
 		     const char *label)
 {
 	if (!adjust_size(&size, count))
-		return NULL;
+		return null_alloc_failed();
 
 	return tal_alloc_(ctx, size, clear, label);
 }

ccan/ccan/tal/tal.h

@@ -11,6 +11,14 @@
 #include <stdbool.h>
 #include <stdarg.h>
 
+/* Define this for better optimization if you never override errfn
+ * to something tat returns */
+#ifdef CCAN_TAL_NEVER_RETURN_NULL
+#define TAL_RETURN_PTR RETURNS_NONNULL
+#else
+#define TAL_RETURN_PTR
+#endif /* CCAN_TAL_NEVER_RETURN_NULL */
+
 /**
  * tal_t - convenient alias for void to mark tal pointers.
  *
@@ -417,7 +425,8 @@ tal_t *tal_parent(const tal_t *ctx);
  * @error_fn: called on errors or NULL (default is abort)
  *
  * The defaults are set up so tal functions never return NULL, but you
- * can override erorr_fn to change that.  error_fn can return, and is
+ * can override error_fn to change that.  error_fn can return (only if
+ * you haven't defined CCAN_TAL_NEVER_RETURN_NULL!), and is
  * called if alloc_fn or resize_fn fail.
  *
  * If any parameter is NULL, that function is unchanged.
@@ -521,9 +530,11 @@ bool tal_set_name_(tal_t *ctx, const char *name, bool literal);
 #define tal_typechk_(ptr, ptype) (ptr)
 #endif
 
-void *tal_alloc_(const tal_t *ctx, size_t bytes, bool clear, const char *label);
+void *tal_alloc_(const tal_t *ctx, size_t bytes, bool clear, const char *label)
+	TAL_RETURN_PTR;
 void *tal_alloc_arr_(const tal_t *ctx, size_t bytes, size_t count, bool clear,
-		     const char *label);
+		     const char *label)
+	TAL_RETURN_PTR;
 
 void *tal_dup_(const tal_t *ctx, const void *p TAKES, size_t size,
 	       size_t n, size_t extra, bool nullok, const char *label);

ccan/ccan/tcon/tcon.h

@@ -147,7 +147,7 @@
  * It evaluates to @x so you can chain it.
  */
 #define tcon_check_ptr(x, canary, expr)				\
-	(sizeof((expr) ? (expr) : &(x)->_tcon[0].canary) ? (x) : (x))
+	(sizeof(0 ? (expr) : &(x)->_tcon[0].canary) ? (x) : (x))
 
 /**
  * tcon_type - the type within a container (or void *)

ccan/ccan/time/test/run-monotonic.c

@@ -7,13 +7,13 @@ int main(void)
 	struct timemono t1, t2;
 	struct timerel t3;
 
-	plan_tests(5);
+	plan_tests(10);
 
 	/* Test time_mono */
 	t1 = time_mono();
 	t2 = time_mono();
 
-	ok1(!time_less_(t2.ts, t1.ts));
+	ok1(!timemono_before(t2, t1));
 
 	t3.ts.tv_sec = 1;
 	t3.ts.tv_nsec = 0;
@@ -24,5 +24,11 @@ int main(void)
 	ok1(timemono_add(t1, t3).ts.tv_sec == t1.ts.tv_sec + 1);
 	ok1(timemono_add(t2, t3).ts.tv_nsec == t2.ts.tv_nsec);
 
+	ok1(timemono_sub(timemono_add(t1, t3), t3).ts.tv_sec == t1.ts.tv_sec);
+	ok1(timemono_sub(timemono_add(t1, t3), t3).ts.tv_nsec == t1.ts.tv_nsec);
+
+	ok1(timemono_after(timemono_add(t1, t3), t1));
+	ok1(!timemono_after(t1, timemono_add(t1, t3)));
+	ok1(!timemono_after(t1, t1));
 	return exit_status();
 }

ccan/ccan/time/time.h

@@ -193,6 +193,23 @@ static inline bool time_greater(struct timerel a, struct timerel b)
 	return time_greater_(a.ts, b.ts);
 }
 
+/**
+ * timemono_after - is a after b?
+ * @a: one monotonic time.
+ * @b: another monotonic time.
+ *
+ * Example:
+ *	static bool timed_out(const struct timemono *start)
+ *	{
+ *	#define TIMEOUT time_from_msec(1000)
+ *		return timemono_after(time_mono(), timemono_add(*start, TIMEOUT));
+ *	}
+ */
+static inline bool timemono_after(struct timemono a, struct timemono b)
+{
+	return time_greater_(a.ts, b.ts);
+}
+
 static inline bool time_less_(struct timespec a, struct timespec b)
 {
 	if (TIME_CHECK(a).tv_sec < TIME_CHECK(b).tv_sec)
@@ -220,6 +237,23 @@ static inline bool time_before(struct timeabs a, struct timeabs b)
 	return time_less_(a.ts, b.ts);
 }
 
+/**
+ * timemono_before - is a before b?
+ * @a: one monotonic time.
+ * @b: another monotonic time.
+ *
+ * Example:
+ *	static bool still_valid(const struct timemono *start)
+ *	{
+ *	#define TIMEOUT time_from_msec(1000)
+ *		return timemono_before(time_mono(), timemono_add(*start, TIMEOUT));
+ *	}
+ */
+static inline bool timemono_before(struct timemono a, struct timemono b)
+{
+	return time_less_(a.ts, b.ts);
+}
+
 /**
  * time_less - is a before b?
  * @a: one relative time.
@@ -404,6 +438,29 @@ static inline struct timeabs timeabs_sub(struct timeabs abs, struct timerel rel)
 	return t;
 }
 
+/**
+ * timemono_sub - subtract a relative time from a monotonic time
+ * @mono: the monotonic time.
+ * @rel: the relative time.
+ *
+ * This returns a well formed struct timemono of @mono - @rel.
+ *
+ * Example:
+ *	// We do one every second.
+ *	static struct timemono previous_time(void)
+ *	{
+ *		return timemono_sub(time_mono(), time_from_msec(1000));
+ *	}
+ */
+static inline struct timemono timemono_sub(struct timemono mono, struct timerel rel)
+{
+	struct timemono t;
+
+	t.ts = time_sub_(mono.ts, rel.ts);
+	return t;
+}
+
+
 static inline struct timespec time_add_(struct timespec a, struct timespec b)
 {
 	struct timespec sum;
@@ -488,6 +545,8 @@ static inline struct timerel timerel_add(struct timerel a, struct timerel b)
  * @div: number to divide it by.
  *
  * Example:
+ *	#include <sys/wait.h>
+ *
  *	// How long does it take to do a fork?
  *	static struct timerel forking_time(void)
  *	{

channeld/Makefile

@@ -95,8 +95,6 @@ CHANNELD_COMMON_OBJS :=				\
 	common/version.o			\
 	common/wire_error.o			\
 	common/wireaddr.o			\
-	gossipd/gossipd_peerd_wiregen.o		\
-	gossipd/gossip_store_wiregen.o		\
 	wire/fromwire.o				\
 	wire/towire.o
 

channeld/channeld_htlc.h

@@ -27,7 +27,7 @@ struct htlc {
 	const u8 *routing;
 
 	/* Blinding (optional). */
-	struct pubkey *blinding;
+	struct pubkey *path_key;
 
 	/* Should we immediately fail this htlc? */
 	bool fail_immediate;
@@ -56,7 +56,8 @@ static inline size_t htlc_hash(u64 id)
 {
 	return siphash24(siphash_seed(), &id, sizeof(id));
 }
-HTABLE_DEFINE_TYPE(struct htlc, htlc_key, htlc_hash, htlc_cmp, htlc_map);
+HTABLE_DEFINE_DUPS_TYPE(struct htlc, htlc_key, htlc_hash, htlc_cmp,
+			htlc_map);
 
 static inline struct htlc *htlc_get(struct htlc_map *htlcs, u64 id, enum side owner)
 {

channeld/channeld_wire.csv

@@ -72,7 +72,6 @@ msgdata,channeld_init,desired_type,channel_type,
 msgdata,channeld_init,dev_disable_commit,?u32,
 msgdata,channeld_init,num_penalty_bases,u32,
 msgdata,channeld_init,pbases,penalty_base,num_penalty_bases
-msgdata,channeld_init,reestablish_only,bool,
 msgdata,channeld_init,experimental_upgrade,bool,
 msgdata,channeld_init,num_inflights,u16,
 msgdata,channeld_init,inflights,inflight,num_inflights
@@ -95,7 +94,7 @@ msgdata,channeld_offer_htlc,amount_msat,amount_msat,
 msgdata,channeld_offer_htlc,cltv_expiry,u32,
 msgdata,channeld_offer_htlc,payment_hash,sha256,
 msgdata,channeld_offer_htlc,onion_routing_packet,u8,1366
-msgdata,channeld_offer_htlc,blinding,?pubkey,
+msgdata,channeld_offer_htlc,path_key,?pubkey,
 
 # Reply; synchronous since IDs have to increment.
 msgtype,channeld_offer_htlc_reply,1104
@@ -210,6 +209,7 @@ msgdata,channeld_splice_init,psbt,wally_psbt,
 msgdata,channeld_splice_init,relative_amount,s64,
 msgdata,channeld_splice_init,feerate_per_kw,u32,
 msgdata,channeld_splice_init,force_feerate,bool,
+msgdata,channeld_splice_init,skip_stfu,bool,
 
 # channeld->master: hello, I started a channel splice open
 msgtype,channeld_splice_confirmed_init,7205
@@ -223,6 +223,7 @@ msgdata,channeld_splice_update,psbt,wally_psbt,
 msgtype,channeld_splice_confirmed_update,7207
 msgdata,channeld_splice_confirmed_update,psbt,wally_psbt,
 msgdata,channeld_splice_confirmed_update,commitments_secured,bool,
+msgdata,channeld_splice_confirmed_update,signatures_secured,bool,
 
 # channeld->master: Lookup a transaction
 msgtype,channeld_splice_lookup_tx,7208
@@ -253,6 +254,7 @@ msgdata,channeld_splice_feerate_error,too_high,bool,
 
 # channeld->master: Add an inflight to the DB
 msgtype,channeld_add_inflight,7216
+msgdata,channeld_add_inflight,remote_funding,pubkey,
 msgdata,channeld_add_inflight,tx_id,bitcoin_txid,
 msgdata,channeld_add_inflight,tx_outnum,u32,
 msgdata,channeld_add_inflight,feerate,u32,
@@ -270,6 +272,7 @@ msgtype,channeld_update_inflight,7219
 msgdata,channeld_update_inflight,psbt,wally_psbt,
 msgdata,channeld_update_inflight,last_tx,?bitcoin_tx,
 msgdata,channeld_update_inflight,last_sig,?bitcoin_signature,
+msgdata,channeld_update_inflight,locked_scid,?short_channel_id,
 
 # channeld->master: A funding error has occured
 msgtype,channeld_splice_funding_error,7220
@@ -285,7 +288,17 @@ msgdata,channeld_splice_state_error,state_error,wirestring,
 msgtype,channeld_splice_abort,7223
 msgdata,channeld_splice_abort,did_i_initiate,bool,
 msgdata,channeld_splice_abort,inflight_outpoint,?bitcoin_outpoint,
-msgdata,channeld_splice_abort,reason,?wirestring,
+msgdata,channeld_splice_abort,reason,wirestring,
+
+# master->channeld: Please enter stfu mode
+msgtype,channeld_stfu,7224
+
+# channeld->master: Entered stfu result
+msgtype,channeld_confirmed_stfu,7225
+msgdata,channeld_confirmed_stfu,available_funds,amount_msat,
+
+# master->channeld: Please enter perform tx_abort
+msgtype,channeld_abort,7226
 
 # Tell peer to shut down channel.
 msgtype,channeld_send_shutdown,1023
@@ -322,8 +335,6 @@ msgdata,channeld_dev_memleak_reply,leak,bool,
 
 # Peer presented proof it was from the future.
 msgtype,channeld_fail_fallen_behind,1028
-# This is NULL if option_static_remotekey.
-msgdata,channeld_fail_fallen_behind,remote_per_commitment_point,?pubkey,
 
 # When we receive announcement_signatures for channel announce
 msgtype,channeld_got_announcement,1017
@@ -349,3 +360,7 @@ msgdata,channeld_upgraded,new_type,channel_type,
 # Tell peer about our latest and greatest blockheight.
 msgtype,channeld_blockheight,1012
 msgdata,channeld_blockheight,blockheight,u32,
+
+# Tell channeld about peer's shachain seed.
+msgtype,channeld_dev_peer_shachain,1013
+msgdata,channeld_dev_peer_shachain,seed,sha256,

channeld/commit_tx.c

@@ -53,7 +53,7 @@ bool commit_tx_amount_trimmed(const struct htlc **htlcs,
 		if (trim(htlcs[i], feerate_per_kw, dust_limit,
 			 option_anchor_outputs, option_anchors_zero_fee_htlc_tx,
 			 side)) {
-			if (!amount_msat_add(amt, *amt, htlcs[i]->amount))
+			if (!amount_msat_accumulate(amt, htlcs[i]->amount))
 				return false;
 		}
 	}

channeld/full_channel.c

@@ -226,7 +226,7 @@ static bool sum_offered_msatoshis(struct amount_msat *total,
 	*total = AMOUNT_MSAT(0);
 	for (i = 0; i < tal_count(htlcs); i++) {
 		if (htlc_owner(htlcs[i]) == side) {
-			if (!amount_msat_add(total, *total, htlcs[i]->amount))
+			if (!amount_msat_accumulate(total, htlcs[i]->amount))
 				return false;
 		}
 	}
@@ -241,7 +241,7 @@ static void add_htlcs(struct bitcoin_tx ***txs,
 {
 	struct bitcoin_outpoint outpoint;
 	u32 htlc_feerate_per_kw;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 
 	if (channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX))
@@ -309,13 +309,17 @@ struct bitcoin_tx **channel_txs(const tal_t *ctx,
 				enum side side,
 				s64 splice_amnt,
 				s64 remote_splice_amnt,
-				int *other_anchor_outnum)
+				int *other_anchor_outnum,
+				const struct pubkey funding_pubkeys[NUM_SIDES])
 {
 	struct bitcoin_tx **txs;
 	const struct htlc **committed;
 	struct keyset keyset;
 	struct amount_msat side_pay, other_side_pay;
 
+	if (!funding_pubkeys)
+		funding_pubkeys = channel->funding_pubkey;
+
 	if (!derive_keyset(per_commitment_point,
 			   &channel->basepoints[side],
 			   &channel->basepoints[!side],
@@ -329,8 +333,8 @@ struct bitcoin_tx **channel_txs(const tal_t *ctx,
 	/* Generating and saving witness script required to spend
 	 * the funding output */
 	*funding_wscript = bitcoin_redeem_2of2(ctx,
-					      &channel->funding_pubkey[side],
-					      &channel->funding_pubkey[!side]);
+					      &funding_pubkeys[side],
+					      &funding_pubkeys[!side]);
 
 	side_pay = channel->view[side].owed[side];
 	other_side_pay = channel->view[side].owed[!side];
@@ -351,8 +355,8 @@ struct bitcoin_tx **channel_txs(const tal_t *ctx,
 	txs[0] = commit_tx(
 	    txs, funding,
 	    funding_sats,
-	    &channel->funding_pubkey[side],
-	    &channel->funding_pubkey[!side],
+	    &funding_pubkeys[side],
+	    &funding_pubkeys[!side],
 	    channel->opener,
 	    channel->config[!side].to_self_delay,
 	    channel->lease_expiry,
@@ -361,15 +365,15 @@ struct bitcoin_tx **channel_txs(const tal_t *ctx,
 	    channel->config[side].dust_limit, side_pay,
 	    other_side_pay, committed, htlcmap, direct_outputs,
 	    commitment_number ^ channel->commitment_number_obscurer,
-	    channel_has(channel, OPT_ANCHOR_OUTPUTS),
+	    channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED),
 	    channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX),
 	    side, other_anchor_outnum);
 
 	/* Set the remote/local pubkeys on the commitment tx psbt */
 	psbt_input_add_pubkey(txs[0]->psbt, 0,
-			      &channel->funding_pubkey[side], false /* is_taproot */);
+			      &funding_pubkeys[side], false /* is_taproot */);
 	psbt_input_add_pubkey(txs[0]->psbt, 0,
-			      &channel->funding_pubkey[!side], false /* is_taproot */);
+			      &funding_pubkeys[!side], false /* is_taproot */);
 
 	add_htlcs(&txs, *htlcmap, channel, &keyset, side);
 
@@ -462,7 +466,7 @@ static struct amount_sat fee_for_htlcs(const struct channel *channel,
 	u32 feerate = channel_feerate(channel, side);
 	struct amount_sat dust_limit = channel->config[side].dust_limit;
 	size_t untrimmed;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 
 	untrimmed = num_untrimmed_htlcs(side, dust_limit, feerate,
@@ -484,7 +488,7 @@ static bool htlc_dust(const struct channel *channel,
 		      struct amount_msat *trim_total)
 {
 	struct amount_sat dust_limit = channel->config[side].dust_limit;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 	struct amount_msat trim_rmvd = AMOUNT_MSAT(0);
 
@@ -522,7 +526,7 @@ static bool htlc_dust(const struct channel *channel,
  *
  * To mostly avoid this situation, at least from our side, we apply an
  * additional constraint when we're opener trying to add an HTLC: make
- * sure we can afford one more HTLC, even if fees increase by 100%.
+ * sure we can afford one more HTLC, even if fees increase.
  *
  * We could do this for the peer, as well, by rejecting their HTLC
  * immediately in this case.  But rejecting a remote HTLC here causes
@@ -544,7 +548,7 @@ static bool local_opener_has_fee_headroom(const struct channel *channel,
 	u32 feerate = channel_feerate(channel, LOCAL);
 	size_t untrimmed;
 	struct amount_sat fee;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 
 	assert(channel->opener == LOCAL);
@@ -559,17 +563,17 @@ static bool local_opener_has_fee_headroom(const struct channel *channel,
 
 	/* Now, how much would it cost us if feerate increases 100% and we added
 	 * another HTLC? */
-	fee = commit_tx_base_fee(2 * feerate, untrimmed + 1,
+	fee = commit_tx_base_fee(marginal_feerate(feerate), untrimmed + 1,
 				 option_anchor_outputs,
 				 option_anchors_zero_fee_htlc_tx);
 	if (amount_msat_greater_eq_sat(remainder, fee))
 		return true;
 
 	status_debug("Adding HTLC would leave us only %s: we need %s for"
-		     " another HTLC if fees increase by 100%% to %uperkw",
+		     " another HTLC if fees increase from %uperkw to %uperkw",
 		     fmt_amount_msat(tmpctx, remainder),
 		     fmt_amount_sat(tmpctx, fee),
-		     feerate + feerate);
+		     feerate, marginal_feerate(feerate));
 	return false;
 }
 
@@ -580,7 +584,7 @@ static enum channel_add_err add_htlc(struct channel *channel,
 				     u32 cltv_expiry,
 				     const struct sha256 *payment_hash,
 				     const u8 routing[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)],
-				     const struct pubkey *blinding TAKES,
+				     const struct pubkey *path_key TAKES,
 				     struct htlc **htlcp,
 				     bool enforce_aggregate_limits,
 				     struct amount_sat *htlc_fee,
@@ -593,7 +597,7 @@ static enum channel_add_err add_htlc(struct channel *channel,
 	const struct htlc **committed, **adding, **removing;
 	const struct channel_view *view;
 	size_t htlc_count;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 	u32 feerate, feerate_ceil;
 
@@ -605,7 +609,7 @@ static enum channel_add_err add_htlc(struct channel *channel,
 	htlc->fail_immediate = false;
 
 	htlc->rhash = *payment_hash;
-	htlc->blinding = tal_dup_or_null(htlc, struct pubkey, blinding);
+	htlc->path_key = tal_dup_or_null(htlc, struct pubkey, path_key);
 	htlc->failed = NULL;
 	htlc->r = NULL;
 	htlc->routing = tal_dup_arr(htlc, u8, routing, TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE), 0);
@@ -647,7 +651,7 @@ static enum channel_add_err add_htlc(struct channel *channel,
 	 *        - SHOULD send a `warning` and close the connection, or send an
 	 *        `error` and fail the channel.
 	 */
-	if (amount_msat_eq(htlc->amount, AMOUNT_MSAT(0))) {
+	if (amount_msat_is_zero(htlc->amount)) {
 		return CHANNEL_ERR_HTLC_BELOW_MINIMUM;
 	}
 	if (amount_msat_less(htlc->amount, channel->config[recipient].htlc_minimum)) {
@@ -898,7 +902,7 @@ enum channel_add_err channel_add_htlc(struct channel *channel,
 				      u32 cltv_expiry,
 				      const struct sha256 *payment_hash,
 				      const u8 routing[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)],
-				      const struct pubkey *blinding TAKES,
+				      const struct pubkey *path_key TAKES,
 				      struct htlc **htlcp,
 				      struct amount_sat *htlc_fee,
 				      bool err_immediate_failures)
@@ -918,7 +922,7 @@ enum channel_add_err channel_add_htlc(struct channel *channel,
 		status_broken("Peer sent out-of-order HTLC ids (is that you, old c-lightning node?)");
 
 	return add_htlc(channel, state, id, amount, cltv_expiry,
-			payment_hash, routing, blinding,
+			payment_hash, routing, path_key,
 			htlcp, true, htlc_fee, err_immediate_failures);
 }
 
@@ -1217,7 +1221,7 @@ u32 approx_max_feerate(const struct channel *channel)
 	struct amount_msat avail;
 	struct balance balance;
 	const struct htlc **committed, **adding, **removing;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 
 	gather_htlcs(tmpctx, channel, !channel->opener,
@@ -1298,7 +1302,7 @@ bool can_opener_afford_feerate(const struct channel *channel, u32 feerate_per_kw
 	struct amount_sat dust_limit = channel->config[!channel->opener].dust_limit;
 	size_t untrimmed;
 	const struct htlc **committed, **adding, **removing;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 	struct balance balance;
 	struct amount_msat available;
@@ -1616,7 +1620,7 @@ bool channel_force_htlcs(struct channel *channel,
 			     htlcs[i]->cltv_expiry,
 			     &htlcs[i]->payment_hash,
 			     htlcs[i]->onion_routing_packet,
-			     htlcs[i]->blinding,
+			     htlcs[i]->path_key,
 			     &htlc, false, NULL, false);
 		if (e != CHANNEL_ERR_ADD_OK) {
 			status_broken("%s HTLC %"PRIu64" failed error %u",
@@ -1654,13 +1658,13 @@ bool channel_force_htlcs(struct channel *channel,
 
 const char *channel_add_err_name(enum channel_add_err e)
 {
-	static char invalidbuf[sizeof("INVALID ") + STR_MAX_CHARS(e)];
+	static char invalidbuf[sizeof("UNKNOWN ") + STR_MAX_CHARS(e)];
 
 	for (size_t i = 0; enum_channel_add_err_names[i].name; i++) {
 		if (enum_channel_add_err_names[i].v == e)
 			return enum_channel_add_err_names[i].name;
 	}
-	snprintf(invalidbuf, sizeof(invalidbuf), "INVALID %i", e);
+	snprintf(invalidbuf, sizeof(invalidbuf), "UNKNOWN %i", e);
 	return invalidbuf;
 }
 

channeld/full_channel.h

@@ -67,6 +67,7 @@ struct channel *new_full_channel(const tal_t *ctx,
  * @local_splice_amnt: how much is being spliced in (or out, if -ve) of local side.
  * @remote_splice_amnt: how much is being spliced in (or out, if -ve) of remote side.
  * @other_anchor_outnum: which output (-1 if none) is the !!side anchor
+ * @funding_pubkeys: The funding pubkeys (specify NULL to use channel's value).
  *
  * Returns the unsigned commitment transaction for the committed state
  * for @side, followed by the htlc transactions in output order and
@@ -84,7 +85,8 @@ struct bitcoin_tx **channel_txs(const tal_t *ctx,
 				enum side side,
 				s64 local_splice_amnt,
 				s64 remote_splice_amnt,
-				int *local_anchor_outnum);
+				int *local_anchor_outnum,
+				const struct pubkey funding_pubkeys[NUM_SIDES]);
 
 /**
  * actual_feerate: what is the actual feerate for the local side.

channeld/inflight.c

@@ -1,6 +1,7 @@
 #include "config.h"
 #include <assert.h>
 #include <bitcoin/psbt.h>
+#include <bitcoin/short_channel_id.h>
 #include <channeld/inflight.h>
 #include <wire/wire.h>
 
@@ -9,6 +10,7 @@ struct inflight *fromwire_inflight(const tal_t *ctx, const u8 **cursor, size_t *
 	struct inflight *inflight = tal(ctx, struct inflight);
 
 	fromwire_bitcoin_outpoint(cursor, max, &inflight->outpoint);
+	fromwire_pubkey(cursor, max, &inflight->remote_funding);
 	inflight->amnt = fromwire_amount_sat(cursor, max);
 	inflight->remote_tx_sigs = fromwire_bool(cursor, max);
 	inflight->psbt = fromwire_wally_psbt(inflight, cursor, max);
@@ -24,6 +26,14 @@ struct inflight *fromwire_inflight(const tal_t *ctx, const u8 **cursor, size_t *
 	}
 	inflight->i_am_initiator = fromwire_bool(cursor, max);
 	inflight->force_sign_first = fromwire_bool(cursor, max);
+	int has_locked_scid = fromwire_u8(cursor, max);
+	if (has_locked_scid) {
+		inflight->locked_scid = tal(inflight, struct short_channel_id);
+		*inflight->locked_scid = fromwire_short_channel_id(cursor, max);
+	}
+	else {
+		inflight->locked_scid = NULL;
+	}
 
 	return inflight;
 }
@@ -31,6 +41,7 @@ struct inflight *fromwire_inflight(const tal_t *ctx, const u8 **cursor, size_t *
 void towire_inflight(u8 **pptr, const struct inflight *inflight)
 {
 	towire_bitcoin_outpoint(pptr, &inflight->outpoint);
+	towire_pubkey(pptr, &inflight->remote_funding);
 	towire_amount_sat(pptr, inflight->amnt);
 	towire_bool(pptr, inflight->remote_tx_sigs);
 	towire_wally_psbt(pptr, inflight->psbt);
@@ -42,17 +53,7 @@ void towire_inflight(u8 **pptr, const struct inflight *inflight)
 	}
 	towire_bool(pptr, inflight->i_am_initiator);
 	towire_bool(pptr, inflight->force_sign_first);
-}
-
-void copy_inflight(struct inflight *dest, struct inflight *src)
-{
-	dest->outpoint = src->outpoint;
-	dest->amnt = src->amnt;
-	dest->remote_tx_sigs = src->remote_tx_sigs;
-	dest->psbt = src->psbt ? clone_psbt(dest, src->psbt): NULL;
-	dest->splice_amnt = src->splice_amnt;
-	dest->last_tx = src->last_tx ? clone_bitcoin_tx(dest, src->last_tx) : NULL;
-	dest->last_sig = src->last_sig;
-	dest->i_am_initiator = src->i_am_initiator;
-	dest->force_sign_first = src->force_sign_first;
+	towire_u8(pptr, inflight->locked_scid ? 1 : 0);
+	if (inflight->locked_scid)
+		towire_short_channel_id(pptr, *inflight->locked_scid);
 }

channeld/inflight.h

@@ -2,11 +2,14 @@
 #define LIGHTNING_CHANNELD_INFLIGHT_H
 
 #include "config.h"
+#include <bitcoin/pubkey.h>
 #include <bitcoin/tx.h>
 #include <common/amount.h>
 
 struct inflight {
+	/* The new channel outpoint */
 	struct bitcoin_outpoint outpoint;
+	struct pubkey remote_funding;
 	struct amount_sat amnt;
 	bool remote_tx_sigs;
 	struct wally_psbt *psbt;
@@ -16,12 +19,10 @@ struct inflight {
 	struct bitcoin_signature last_sig;
 	bool i_am_initiator;
 	bool force_sign_first;
-	bool is_locked;
+	struct short_channel_id *locked_scid;
 };
 
 struct inflight *fromwire_inflight(const tal_t *ctx, const u8 **cursor, size_t *max);
 void towire_inflight(u8 **pptr, const struct inflight *inflight);
 
-void copy_inflight(struct inflight *dest, struct inflight *src);
-
 #endif /* LIGHTNING_CHANNELD_INFLIGHT_H */

channeld/splice.c

@@ -11,6 +11,7 @@ struct splice_state *splice_state_new(const tal_t *ctx)
 	splice_state->locked_ready[REMOTE] = false;
 	splice_state->await_commitment_succcess = false;
 	splice_state->inflights = NULL;
+	splice_state->remote_locked_txid = NULL;
 
 	return splice_state;
 }
@@ -31,6 +32,8 @@ struct splicing *splicing_new(const tal_t *ctx)
 	splicing->received_tx_complete = false;
 	splicing->sent_tx_complete = false;
 	splicing->tx_sig_msg = NULL;
+	splicing->inws = NULL;
+	splicing->their_sig = NULL;
 
 	return splicing;
 }

channeld/splice.h

@@ -21,6 +21,8 @@ struct splice_state {
 	bool await_commitment_succcess;
 	/* The txid of which splice inflight was confirmed */
 	struct bitcoin_txid locked_txid;
+	/* The txid our peer locked their splice on */
+	struct bitcoin_txid *remote_locked_txid;
 	/* The number of splices that are active (awaiting confirmation) */
 	u32 count;
 };
@@ -31,6 +33,8 @@ struct splice_state *splice_state_new(const tal_t *ctx);
 /* An active splice negotiation. Born when splice beings and dies when a splice
  * negotation has finished */
 struct splicing {
+	/* The remote side's rotated funding pubkey */
+	struct pubkey remote_funding_pubkey;
 	/* The opener side's relative balance change */
 	s64 opener_relative;
 	/* The accepter side's relative balance change */
@@ -53,6 +57,10 @@ struct splicing {
 	bool sent_tx_complete;
 	/* If our peer signs early, we allow that and cache it here */
 	const u8 *tx_sig_msg;
+	/* The witness stack data received by peer */
+	struct witness **inws;
+	/* Their channel funding signature */
+	struct bitcoin_signature *their_sig;
 };
 
 /* Sets `splice` items to default values */

channeld/test/run-commit_tx.c

@@ -13,6 +13,7 @@ static bool print_superverbose;
 #include <ccan/err/err.h>
 #include <ccan/str/hex/hex.h>
 #include <common/channel_id.h>
+#include <common/daemon.h>
 #include <common/key_derive.h>
 #include <common/setup.h>
 #include <common/status.h>
@@ -31,6 +32,12 @@ bool fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
 /* Generated stub for fromwire_node_id */
 void fromwire_node_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct node_id *id UNNEEDED)
 { fprintf(stderr, "fromwire_node_id called!\n"); abort(); }
+/* Generated stub for pubkey_from_node_id */
+bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
+{ fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
+/* Generated stub for send_backtrace */
+void send_backtrace(const char *why UNNEEDED)
+{ fprintf(stderr, "send_backtrace called!\n"); abort(); }
 /* Generated stub for status_fmt */
 void status_fmt(enum log_level level UNNEEDED,
 		const struct node_id *peer UNNEEDED,
@@ -564,7 +571,7 @@ int main(int argc, const char *argv[])
 	 * In the following:
 	 *  - *local* transactions are considered, which implies that all
 	 *    payments to *local* are delayed.
-	 *  - It's assumed that *local* is the funder.
+	 *  - It's assumed that *local* is the opener.
 	 *  - Private keys are displayed as 32 bytes plus a trailing 1
 	 *    (Bitcoin's convention for "compressed" private keys, i.e. keys for
 	 *    which the public key is compressed).

channeld/test/run-full_channel.c

@@ -1,12 +1,15 @@
 #include "config.h"
 #include "../../common/blockheight_states.c"
 #include "../../common/channel_id.c"
+/* Normally fee_states does not allow feerate < 253 */
+#define TEST_ALLOW_ZERO_FEERATE 1
 #include "../../common/fee_states.c"
 #include "../../common/initial_channel.c"
 #include "../../common/keyset.c"
 #include "../full_channel.c"
 #include "../commit_tx.c"
 #include <ccan/err/err.h>
+#include <common/daemon.h>
 #include <common/setup.h>
 
 /* AUTOGENERATED MOCKS START */
@@ -22,6 +25,12 @@ void memleak_add_helper_(const tal_t *p UNNEEDED, void (*cb)(struct htable *memt
 /* Generated stub for memleak_scan_htable */
 void memleak_scan_htable(struct htable *memtable UNNEEDED, const struct htable *ht UNNEEDED)
 { fprintf(stderr, "memleak_scan_htable called!\n"); abort(); }
+/* Generated stub for pubkey_from_node_id */
+bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
+{ fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
+/* Generated stub for send_backtrace */
+void send_backtrace(const char *why UNNEEDED)
+{ fprintf(stderr, "send_backtrace called!\n"); abort(); }
 /* Generated stub for status_failed */
 void status_failed(enum status_failreason code UNNEEDED,
 		   const char *fmt UNNEEDED, ...)
@@ -381,7 +390,7 @@ int main(int argc, const char *argv[])
 	 * In the following:
 	 *  - *local* transactions are considered, which implies that all
 	 *    payments to *local* are delayed.
-	 *  - It's assumed that *local* is the funder.
+	 *  - It's assumed that *local* is the opener.
 	 *  - Private keys are displayed as 32 bytes plus a trailing 1
 	 *    (Bitcoin's convention for "compressed" private keys, i.e. keys for
 	 *    which the public key is compressed).
@@ -488,7 +497,7 @@ int main(int argc, const char *argv[])
 				    &localbase, &remotebase,
 				    &local_funding_pubkey,
 				    &remote_funding_pubkey,
-				    take(channel_type_none(NULL)), false, LOCAL);
+				    take(channel_type_static_remotekey(NULL)), false, LOCAL);
 	rchannel = new_full_channel(tmpctx, &cid,
 				    &funding, 0,
 				    take(new_height_states(NULL, REMOTE, &blockheight)),
@@ -501,7 +510,7 @@ int main(int argc, const char *argv[])
 				    &remotebase, &localbase,
 				    &remote_funding_pubkey,
 				    &local_funding_pubkey,
-				    take(channel_type_none(NULL)), false, REMOTE);
+				    take(channel_type_static_remotekey(NULL)), false, REMOTE);
 
 	/* BOLT #3:
 	 *
@@ -509,19 +518,23 @@ int main(int argc, const char *argv[])
 	 */
 	local_per_commitment_point = pubkey_from_hex("025f7117a78150fe2ef97db7cfc83bd57b2e2c0d0dd25eaf467a4a1c2a45ce1486");
 	/* BOLT #3:
-	 * localpubkey: 030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e7
-	 * remotepubkey: 0394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b
+	 * local_funding_pubkey: 023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb
+	 * remote_funding_pubkey: 030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c1
+	 * local_privkey: bb13b121cdc357cd2e608b0aea294afca36e2b34cf958e2e6451a2f27469449101
+	 * local_htlcpubkey: 030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e7
+	 * remote_htlcpubkey: 0394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b
 	 * local_delayedpubkey: 03fd5960528dc152014952efdb702a88f71e3c1653b2314431701ec77e57fde83c
 	 * local_revocation_pubkey: 0212a140cd0c6539d07cd08dfe09984dec3251ea808b892efeac3ede9402bf2b19
 	 */
-	keyset.self_payment_key = pubkey_from_hex("030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e7");
-	keyset.other_payment_key = pubkey_from_hex("0394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b");
+
+	/* FIXME: Above is wrong for static remotekey!  local/remote pubkey are just the basepoints */
+	keyset.self_payment_key = localbase.payment;
+	keyset.other_payment_key = remotebase.payment;
 	keyset.self_delayed_payment_key = pubkey_from_hex("03fd5960528dc152014952efdb702a88f71e3c1653b2314431701ec77e57fde83c");
 	keyset.self_revocation_key = pubkey_from_hex("0212a140cd0c6539d07cd08dfe09984dec3251ea808b892efeac3ede9402bf2b19");
 
-	/* FIXME: Update bolt */
-	keyset.self_htlc_key = keyset.self_payment_key;
-	keyset.other_htlc_key = keyset.other_payment_key;
+	keyset.self_htlc_key = pubkey_from_hex("030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e7");
+	keyset.other_htlc_key = pubkey_from_hex("0394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b");
 
 	raw_tx = commit_tx(tmpctx,
 			   &funding,
@@ -542,7 +555,7 @@ int main(int argc, const char *argv[])
 	txs = channel_txs(tmpctx, &funding, funding_amount,
 			  &htlc_map, NULL, &funding_wscript_alt,
 			  lchannel, &local_per_commitment_point, 42, LOCAL, 0, 0,
-			  &local_anchor);
+			  &local_anchor, NULL);
 	assert(tal_count(txs) == 1);
 	assert(tal_count(htlc_map) == 2);
 	assert(scripteq(funding_wscript_alt, funding_wscript));
@@ -551,7 +564,7 @@ int main(int argc, const char *argv[])
 	txs2 = channel_txs(tmpctx, &funding, funding_amount,
 			   &htlc_map, NULL, &funding_wscript,
 			   rchannel, &local_per_commitment_point, 42, REMOTE, 0, 0,
-			   &local_anchor);
+			   &local_anchor, NULL);
 	txs_must_be_eq(txs, txs2);
 
 	/* BOLT #3:
@@ -580,12 +593,12 @@ int main(int argc, const char *argv[])
 	txs = channel_txs(tmpctx, &funding, funding_amount,
 			  &htlc_map, NULL, &funding_wscript,
 			  lchannel, &local_per_commitment_point, 42, LOCAL, 0, 0,
-			  &local_anchor);
+			  &local_anchor, NULL);
 	assert(tal_count(txs) == 1);
 	txs2 = channel_txs(tmpctx, &funding, funding_amount,
 			   &htlc_map, NULL, &funding_wscript,
 			   rchannel, &local_per_commitment_point, 42, REMOTE, 0, 0,
-			   &local_anchor);
+			   &local_anchor, NULL);
 	txs_must_be_eq(txs, txs2);
 
 	update_feerate(lchannel, feerate_per_kw[LOCAL]);
@@ -602,56 +615,56 @@ int main(int argc, const char *argv[])
 	txs = channel_txs(tmpctx, &funding, funding_amount,
 			  &htlc_map, NULL, &funding_wscript,
 			  lchannel, &local_per_commitment_point, 42, LOCAL, 0, 0,
-			  &local_anchor);
+			  &local_anchor, NULL);
 	assert(tal_count(txs) == 6);
 	txs2 = channel_txs(tmpctx, &funding, funding_amount,
 			   &htlc_map, NULL, &funding_wscript,
 			   rchannel, &local_per_commitment_point, 42, REMOTE, 0, 0,
-			   &local_anchor);
+			   &local_anchor, NULL);
 	txs_must_be_eq(txs, txs2);
 
 	/* FIXME: Compare signatures! */
 	/* BOLT #3:
 	 *
-	 *     htlc_success_tx (htlc #0): 020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219700000000000000000001e8030000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e050047304402206a6e59f18764a5bf8d4fa45eebc591566689441229c918b480fb2af8cc6a4aeb02205248f273be447684b33e3c8d1d85a8e0ca9fa0bae9ae33f0527ada9c162919a60147304402207cb324fa0de88f452ffa9389678127ebcf4cabe1dd848b8e076c1a1962bf34720220116ed922b12311bd602d67e60d2529917f21c5b82f25ff6506c0f87886b4dfd5012000000000000000000000000000000000000000000000000000000000000000008a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a914b8bcb07f6344b42ab04250c86a6e8b75d3fdbbc688527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f401b175ac686800000000
+	 *     htlc_success_tx (htlc #0): 02000000000101ab84ff284f162cfbfef241f853b47d4368d171f9e2a1445160cd591c4c7d882b00000000000000000001e8030000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0500483045022100d9e29616b8f3959f1d3d7f7ce893ffedcdc407717d0de8e37d808c91d3a7c50d022078c3033f6d00095c8720a4bc943c1b45727818c082e4e3ddbc6d3116435b624b014730440220636de5682ef0c5b61f124ec74e8aa2461a69777521d6998295dcea36bc3338110220165285594b23c50b28b82df200234566628a27bcd17f7f14404bd865354eb3ce012000000000000000000000000000000000000000000000000000000000000000008a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a914b8bcb07f6344b42ab04250c86a6e8b75d3fdbbc688527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f401b175ac686800000000
 	 */
-	raw_tx = tx_from_hex(tmpctx, "020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219700000000000000000001e8030000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e050047304402206a6e59f18764a5bf8d4fa45eebc591566689441229c918b480fb2af8cc6a4aeb02205248f273be447684b33e3c8d1d85a8e0ca9fa0bae9ae33f0527ada9c162919a60147304402207cb324fa0de88f452ffa9389678127ebcf4cabe1dd848b8e076c1a1962bf34720220116ed922b12311bd602d67e60d2529917f21c5b82f25ff6506c0f87886b4dfd5012000000000000000000000000000000000000000000000000000000000000000008a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a914b8bcb07f6344b42ab04250c86a6e8b75d3fdbbc688527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f401b175ac686800000000");
+	raw_tx = tx_from_hex(tmpctx, "02000000000101ab84ff284f162cfbfef241f853b47d4368d171f9e2a1445160cd591c4c7d882b00000000000000000001e8030000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0500483045022100d9e29616b8f3959f1d3d7f7ce893ffedcdc407717d0de8e37d808c91d3a7c50d022078c3033f6d00095c8720a4bc943c1b45727818c082e4e3ddbc6d3116435b624b014730440220636de5682ef0c5b61f124ec74e8aa2461a69777521d6998295dcea36bc3338110220165285594b23c50b28b82df200234566628a27bcd17f7f14404bd865354eb3ce012000000000000000000000000000000000000000000000000000000000000000008a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a914b8bcb07f6344b42ab04250c86a6e8b75d3fdbbc688527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f401b175ac686800000000");
 	raw_tx->chainparams = chainparams;
 	bitcoin_tx_input_set_witness(raw_tx, 0, NULL);
 	tx_must_be_eq(raw_tx, txs[1]);
 
 	/* BOLT #3:
 	 *
-	 *     htlc_timeout_tx (htlc #2): 020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219701000000000000000001d0070000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0500483045022100d5275b3619953cb0c3b5aa577f04bc512380e60fa551762ce3d7a1bb7401cff9022037237ab0dac3fe100cde094e82e2bed9ba0ed1bb40154b48e56aa70f259e608b01483045022100c89172099507ff50f4c925e6c5150e871fb6e83dd73ff9fbb72f6ce829a9633f02203a63821d9162e99f9be712a68f9e589483994feae2661e4546cd5b6cec007be501008576a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a914b43e1b38138a41b37f7cd9a1d274bc63e3a9b5d188ac6868f6010000
+	 *     htlc_timeout_tx (htlc #2): 02000000000101ab84ff284f162cfbfef241f853b47d4368d171f9e2a1445160cd591c4c7d882b01000000000000000001d0070000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e05004730440220649fe8b20e67e46cbb0d09b4acea87dbec001b39b08dee7bdd0b1f03922a8640022037c462dff79df501cecfdb12ea7f4de91f99230bb544726f6e04527b1f89600401483045022100803159dee7935dba4a1d36a61055ce8fd62caa528573cc221ae288515405a252022029c59e7cffce374fe860100a4a63787e105c3cf5156d40b12dd53ff55ac8cf3f01008576a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a914b43e1b38138a41b37f7cd9a1d274bc63e3a9b5d188ac6868f6010000
 	 */
-	raw_tx = tx_from_hex(tmpctx, "020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219701000000000000000001d0070000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0500483045022100d5275b3619953cb0c3b5aa577f04bc512380e60fa551762ce3d7a1bb7401cff9022037237ab0dac3fe100cde094e82e2bed9ba0ed1bb40154b48e56aa70f259e608b01483045022100c89172099507ff50f4c925e6c5150e871fb6e83dd73ff9fbb72f6ce829a9633f02203a63821d9162e99f9be712a68f9e589483994feae2661e4546cd5b6cec007be501008576a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a914b43e1b38138a41b37f7cd9a1d274bc63e3a9b5d188ac6868f6010000");
+	raw_tx = tx_from_hex(tmpctx, "02000000000101ab84ff284f162cfbfef241f853b47d4368d171f9e2a1445160cd591c4c7d882b01000000000000000001d0070000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e05004730440220649fe8b20e67e46cbb0d09b4acea87dbec001b39b08dee7bdd0b1f03922a8640022037c462dff79df501cecfdb12ea7f4de91f99230bb544726f6e04527b1f89600401483045022100803159dee7935dba4a1d36a61055ce8fd62caa528573cc221ae288515405a252022029c59e7cffce374fe860100a4a63787e105c3cf5156d40b12dd53ff55ac8cf3f01008576a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a914b43e1b38138a41b37f7cd9a1d274bc63e3a9b5d188ac6868f6010000");
 	raw_tx->chainparams = chainparams;
 	bitcoin_tx_input_set_witness(raw_tx, 0, NULL);
 	tx_must_be_eq(raw_tx, txs[2]);
 
 	/* BOLT #3:
 	 *
-	 *     htlc_success_tx (htlc #1): 020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219702000000000000000001d0070000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e050047304402201b63ec807771baf4fdff523c644080de17f1da478989308ad13a58b51db91d360220568939d38c9ce295adba15665fa68f51d967e8ed14a007b751540a80b325f20201483045022100def389deab09cee69eaa1ec14d9428770e45bcbe9feb46468ecf481371165c2f022015d2e3c46600b2ebba8dcc899768874cc6851fd1ecb3fffd15db1cc3de7e10da012001010101010101010101010101010101010101010101010101010101010101018a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a9144b6b2e5444c2639cc0fb7bcea5afba3f3cdce23988527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f501b175ac686800000000
+	 *     htlc_success_tx (htlc #1): 02000000000101ab84ff284f162cfbfef241f853b47d4368d171f9e2a1445160cd591c4c7d882b02000000000000000001d0070000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e05004730440220770fc321e97a19f38985f2e7732dd9fe08d16a2efa4bcbc0429400a447faf49102204d40b417f3113e1b0944ae0986f517564ab4acd3d190503faf97a6e420d4335201483045022100a437cc2ce77400ecde441b3398fea3c3ad8bdad8132be818227fe3c5b8345989022069d45e7fa0ae551ec37240845e2c561ceb2567eacf3076a6a43a502d05865faa012001010101010101010101010101010101010101010101010101010101010101018a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a9144b6b2e5444c2639cc0fb7bcea5afba3f3cdce23988527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f501b175ac686800000000
 	 */
-	raw_tx = tx_from_hex(tmpctx, "020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219702000000000000000001d0070000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e050047304402201b63ec807771baf4fdff523c644080de17f1da478989308ad13a58b51db91d360220568939d38c9ce295adba15665fa68f51d967e8ed14a007b751540a80b325f20201483045022100def389deab09cee69eaa1ec14d9428770e45bcbe9feb46468ecf481371165c2f022015d2e3c46600b2ebba8dcc899768874cc6851fd1ecb3fffd15db1cc3de7e10da012001010101010101010101010101010101010101010101010101010101010101018a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a9144b6b2e5444c2639cc0fb7bcea5afba3f3cdce23988527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f501b175ac686800000000");
+	raw_tx = tx_from_hex(tmpctx, "02000000000101ab84ff284f162cfbfef241f853b47d4368d171f9e2a1445160cd591c4c7d882b02000000000000000001d0070000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e05004730440220770fc321e97a19f38985f2e7732dd9fe08d16a2efa4bcbc0429400a447faf49102204d40b417f3113e1b0944ae0986f517564ab4acd3d190503faf97a6e420d4335201483045022100a437cc2ce77400ecde441b3398fea3c3ad8bdad8132be818227fe3c5b8345989022069d45e7fa0ae551ec37240845e2c561ceb2567eacf3076a6a43a502d05865faa012001010101010101010101010101010101010101010101010101010101010101018a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a9144b6b2e5444c2639cc0fb7bcea5afba3f3cdce23988527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f501b175ac686800000000");
 	raw_tx->chainparams = chainparams;
 	bitcoin_tx_input_set_witness(raw_tx, 0, NULL);
 	tx_must_be_eq(raw_tx, txs[3]);
 
 	/* BOLT #3:
 	 *
-	 *     htlc_timeout_tx (htlc #3): 020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219703000000000000000001b80b0000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0500483045022100daee1808f9861b6c3ecd14f7b707eca02dd6bdfc714ba2f33bc8cdba507bb182022026654bf8863af77d74f51f4e0b62d461a019561bb12acb120d3f7195d148a554014730440220643aacb19bbb72bd2b635bc3f7375481f5981bace78cdd8319b2988ffcc6704202203d27784ec8ad51ed3bd517a05525a5139bb0b755dd719e0054332d186ac0872701008576a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a9148a486ff2e31d6158bf39e2608864d63fefd09d5b88ac6868f7010000
+	 *     htlc_timeout_tx (htlc #3): 02000000000101ab84ff284f162cfbfef241f853b47d4368d171f9e2a1445160cd591c4c7d882b03000000000000000001b80b0000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e050047304402207bcbf4f60a9829b05d2dbab84ed593e0291836be715dc7db6b72a64caf646af802201e489a5a84f7c5cc130398b841d138d031a5137ac8f4c49c770a4959dc3c13630147304402203121d9b9c055f354304b016a36662ee99e1110d9501cb271b087ddb6f382c2c80220549882f3f3b78d9c492de47543cb9a697cecc493174726146536c5954dac748701008576a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a9148a486ff2e31d6158bf39e2608864d63fefd09d5b88ac6868f7010000
 	 */
-	raw_tx = tx_from_hex(tmpctx, "020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219703000000000000000001b80b0000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0500483045022100daee1808f9861b6c3ecd14f7b707eca02dd6bdfc714ba2f33bc8cdba507bb182022026654bf8863af77d74f51f4e0b62d461a019561bb12acb120d3f7195d148a554014730440220643aacb19bbb72bd2b635bc3f7375481f5981bace78cdd8319b2988ffcc6704202203d27784ec8ad51ed3bd517a05525a5139bb0b755dd719e0054332d186ac0872701008576a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a9148a486ff2e31d6158bf39e2608864d63fefd09d5b88ac6868f7010000");
+	raw_tx = tx_from_hex(tmpctx, "02000000000101ab84ff284f162cfbfef241f853b47d4368d171f9e2a1445160cd591c4c7d882b03000000000000000001b80b0000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e050047304402207bcbf4f60a9829b05d2dbab84ed593e0291836be715dc7db6b72a64caf646af802201e489a5a84f7c5cc130398b841d138d031a5137ac8f4c49c770a4959dc3c13630147304402203121d9b9c055f354304b016a36662ee99e1110d9501cb271b087ddb6f382c2c80220549882f3f3b78d9c492de47543cb9a697cecc493174726146536c5954dac748701008576a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c820120876475527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae67a9148a486ff2e31d6158bf39e2608864d63fefd09d5b88ac6868f7010000");
 	raw_tx->chainparams = chainparams;
 	bitcoin_tx_input_set_witness(raw_tx, 0, NULL);
 	tx_must_be_eq(raw_tx, txs[4]);
 
 	/* BOLT #3:
 	 *
-	 *     htlc_success_tx (htlc #4): 020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219704000000000000000001a00f0000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e050047304402207e0410e45454b0978a623f36a10626ef17b27d9ad44e2760f98cfa3efb37924f0220220bd8acd43ecaa916a80bd4f919c495a2c58982ce7c8625153f8596692a801d014730440220549e80b4496803cbc4a1d09d46df50109f546d43fbbf86cd90b174b1484acd5402205f12a4f995cb9bded597eabfee195a285986aa6d93ae5bb72507ebc6a4e2349e012004040404040404040404040404040404040404040404040404040404040404048a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a91418bc1a114ccf9c052d3d23e28d3b0a9d1227434288527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f801b175ac686800000000
+	 *     htlc_success_tx (htlc #4): 02000000000101ab84ff284f162cfbfef241f853b47d4368d171f9e2a1445160cd591c4c7d882b04000000000000000001a00f0000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0500473044022076dca5cb81ba7e466e349b7128cdba216d4d01659e29b96025b9524aaf0d1899022060de85697b88b21c749702b7d2cfa7dfeaa1f472c8f1d7d9c23f2bf968464b8701483045022100d9080f103cc92bac15ec42464a95f070c7fb6925014e673ee2ea1374d36a7f7502200c65294d22eb20d48564954d5afe04a385551919d8b2ddb4ae2459daaeee1d95012004040404040404040404040404040404040404040404040404040404040404048a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a91418bc1a114ccf9c052d3d23e28d3b0a9d1227434288527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f801b175ac686800000000
 	 */
-	raw_tx = tx_from_hex(tmpctx, "020000000001018154ecccf11a5fb56c39654c4deb4d2296f83c69268280b94d021370c94e219704000000000000000001a00f0000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e050047304402207e0410e45454b0978a623f36a10626ef17b27d9ad44e2760f98cfa3efb37924f0220220bd8acd43ecaa916a80bd4f919c495a2c58982ce7c8625153f8596692a801d014730440220549e80b4496803cbc4a1d09d46df50109f546d43fbbf86cd90b174b1484acd5402205f12a4f995cb9bded597eabfee195a285986aa6d93ae5bb72507ebc6a4e2349e012004040404040404040404040404040404040404040404040404040404040404048a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a91418bc1a114ccf9c052d3d23e28d3b0a9d1227434288527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f801b175ac686800000000");
+	raw_tx = tx_from_hex(tmpctx, "02000000000101ab84ff284f162cfbfef241f853b47d4368d171f9e2a1445160cd591c4c7d882b04000000000000000001a00f0000000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e0500473044022076dca5cb81ba7e466e349b7128cdba216d4d01659e29b96025b9524aaf0d1899022060de85697b88b21c749702b7d2cfa7dfeaa1f472c8f1d7d9c23f2bf968464b8701483045022100d9080f103cc92bac15ec42464a95f070c7fb6925014e673ee2ea1374d36a7f7502200c65294d22eb20d48564954d5afe04a385551919d8b2ddb4ae2459daaeee1d95012004040404040404040404040404040404040404040404040404040404040404048a76a91414011f7254d96b819c76986c277d115efce6f7b58763ac67210394854aa6eab5b2a8122cc726e9dded053a2184d88256816826d6231c068d4a5b7c8201208763a91418bc1a114ccf9c052d3d23e28d3b0a9d1227434288527c21030d417a46946384f88d5f3337267c5e579765875dc4daca813e21734b140639e752ae677502f801b175ac686800000000");
 	raw_tx->chainparams = chainparams;
 	bitcoin_tx_input_set_witness(raw_tx, 0, NULL);
 	tx_must_be_eq(raw_tx, txs[5]);
@@ -682,14 +695,14 @@ int main(int argc, const char *argv[])
 				  &htlc_map, NULL, &funding_wscript,
 				  lchannel, &local_per_commitment_point, 42,
 				  LOCAL, 0, 0,
-				  &local_anchor);
+				  &local_anchor, NULL);
 		tx_must_be_eq(txs[0], raw_tx);
 
 		txs2 = channel_txs(tmpctx, &funding, funding_amount,
 				   &htlc_map, NULL, &funding_wscript,
 				   rchannel, &local_per_commitment_point,
 				   42, REMOTE, 0, 0,
-				   &local_anchor);
+				   &local_anchor, NULL);
 		txs_must_be_eq(txs, txs2);
 	}
 

cli/lightning-cli.c

@@ -179,13 +179,9 @@ static void human_help(char *buffer, const jsmntok_t *result)
 	unsigned int i;
 	/* `curr` is used as a temporary token */
 	const jsmntok_t *curr;
-	const char *prev_cat;
 	/* Contains all commands objects, which have the following structure :
 	 * {
-	 *     "command": "The command name and usage",
-	 *     "category": "The command category",
-	 *     "description": "The command's description",
-	 *     "verbose": "The command's detailed description"
+	 *     "command": "The command name and usage"
 	 * }
 	 */
 	const jsmntok_t * help_array = json_get_member(buffer, result, "help");
@@ -198,22 +194,11 @@ static void human_help(char *buffer, const jsmntok_t *result)
 
 	asort(help, tal_count(help), compare_help, buffer);
 
-	prev_cat = "";
 	for (i = 0; i < tal_count(help); i++) {
-		const jsmntok_t *category, *command, *desc;
-
-		category = json_get_member(buffer, help[i], "category");
-		if (category && !json_tok_streq(buffer, category, prev_cat)) {
-			prev_cat = json_strdup(help, buffer, category);
-			printf("=== %s ===\n\n", prev_cat);
-		}
-
+		const jsmntok_t *command;
 		command = json_get_member(buffer, help[i], "command");
-		desc = json_get_member(buffer, help[i], "description");
-		printf("%.*s\n",
-		       command->end - command->start, buffer + command->start);
-		printf("    %.*s\n\n",
-		       desc->end - desc->start, buffer + desc->start);
+		human_readable(buffer, command, '\n');
+		printf("\n");
 	}
 	tal_free(help);
 
@@ -309,7 +294,7 @@ try_exec_man (const char *page, char *relative_to) {
 	case 0:
 		/* child, run man command. */
 		if (relative_to != NULL) {
-			page = tal_fmt(page, "%s/../doc/%s.7", relative_to, page);
+			page = tal_fmt(page, "%s/%s.7", relative_to, page);
 			execlp("man", "man", "-l", page, (char *)NULL);
 		}
 		else {
@@ -716,12 +701,18 @@ int main(int argc, char *argv[])
 		command = argv[2];
 		char *page = tal_fmt(ctx, "lightning-%s", command);
 
+		/* Try to find the page in the MANPATH and PATH. */
 		try_exec_man(page, NULL);
 
 		/* Try to find the page relative to this executable.
 		 * This handles the common scenario where lightning-cli
 		 * was built from source and hasn't been installed yet */
-		try_exec_man(page, dirname(argv[0]));
+		const char *dir = dirname(argv[0]);
+		try_exec_man(page, tal_fmt(page, "%s/../doc", dir));
+
+		/* Try to find the page relative to this executable, but in the
+		 * install directory hierarchy. */
+		try_exec_man(page, tal_fmt(page, "%s/../share/man/man7", dir));
 
 		tal_free(page);
 	}
@@ -875,10 +866,8 @@ int main(int argc, char *argv[])
 		default:
 			if (handle_notify(resp, toks, notification_level,
 					  &last_was_progress)) {
-				/* +2 for \n\n */
-				size_t len = toks[0].end - toks[0].start + 2;
-				memmove(resp, resp + len, off - len);
-				off -= len;
+				memmove(resp, resp + toks[0].end, off - toks[0].end);
+				off -= toks[0].end;
 				jsmn_init(&parser);
 				toks[0].type = JSMN_UNDEFINED;
 				/* Don't force another read! */

cli/test/run-human-mode.c

@@ -4,6 +4,7 @@
 #include <common/bigsize.h>
 #include <common/channel_id.h>
 #include <common/configvar.h>
+#include <common/daemon.h>
 #include <common/json_stream.h>
 #include <common/node_id.h>
 #include <common/setup.h>
@@ -89,6 +90,12 @@ const char *log_level_name(enum log_level level UNNEEDED)
 bool log_level_parse(const char *levelstr UNNEEDED, size_t len UNNEEDED,
 		     enum log_level *level UNNEEDED)
 { fprintf(stderr, "log_level_parse called!\n"); abort(); }
+/* Generated stub for pubkey_from_node_id */
+bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
+{ fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
+/* Generated stub for send_backtrace */
+void send_backtrace(const char *why UNNEEDED)
+{ fprintf(stderr, "send_backtrace called!\n"); abort(); }
 /* Generated stub for towire_amount_msat */
 void towire_amount_msat(u8 **pptr UNNEEDED, const struct amount_msat msat UNNEEDED)
 { fprintf(stderr, "towire_amount_msat called!\n"); abort(); }

cli/test/run-large-input.c

@@ -4,6 +4,7 @@
 #include <common/bigsize.h>
 #include <common/channel_id.h>
 #include <common/configvar.h>
+#include <common/daemon.h>
 #include <common/json_stream.h>
 #include <common/node_id.h>
 #include <common/setup.h>
@@ -89,6 +90,12 @@ const char *log_level_name(enum log_level level UNNEEDED)
 bool log_level_parse(const char *levelstr UNNEEDED, size_t len UNNEEDED,
 		     enum log_level *level UNNEEDED)
 { fprintf(stderr, "log_level_parse called!\n"); abort(); }
+/* Generated stub for pubkey_from_node_id */
+bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
+{ fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
+/* Generated stub for send_backtrace */
+void send_backtrace(const char *why UNNEEDED)
+{ fprintf(stderr, "send_backtrace called!\n"); abort(); }
 /* Generated stub for towire_amount_msat */
 void towire_amount_msat(u8 **pptr UNNEEDED, const struct amount_msat msat UNNEEDED)
 { fprintf(stderr, "towire_amount_msat called!\n"); abort(); }

cli/test/run-remove-hint.c

@@ -4,6 +4,7 @@
 #include <common/bigsize.h>
 #include <common/channel_id.h>
 #include <common/configvar.h>
+#include <common/daemon.h>
 #include <common/json_stream.h>
 #include <common/node_id.h>
 #include <common/setup.h>
@@ -92,6 +93,12 @@ const char *log_level_name(enum log_level level UNNEEDED)
 bool log_level_parse(const char *levelstr UNNEEDED, size_t len UNNEEDED,
 		     enum log_level *level UNNEEDED)
 { fprintf(stderr, "log_level_parse called!\n"); abort(); }
+/* Generated stub for pubkey_from_node_id */
+bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
+{ fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
+/* Generated stub for send_backtrace */
+void send_backtrace(const char *why UNNEEDED)
+{ fprintf(stderr, "send_backtrace called!\n"); abort(); }
 /* Generated stub for towire_amount_msat */
 void towire_amount_msat(u8 **pptr UNNEEDED, const struct amount_msat msat UNNEEDED)
 { fprintf(stderr, "towire_amount_msat called!\n"); abort(); }

cln-grpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cln-grpc"
-version = "0.1.8"
+version = "0.4.1"
 edition = "2021"
 license = "MIT"
 description = "The Core Lightning API as grpc primitives. Provides the bindings used to expose the API over the network."
@@ -15,16 +15,21 @@ server = ["cln-rpc"]
 [dependencies]
 anyhow = "1.0"
 log = "0.4"
-cln-rpc = { path="../cln-rpc/", version = "^0.1", optional = true }
-tonic = { version = "0.8", features = ["tls", "transport"] }
-prost = "0.11"
+cln-rpc = { path = "../cln-rpc/", version = "0.4", optional = true }
+cfg-if = "1.0"
 serde = { version = "1.0", features = ["derive"] }
+tonic = { version = "0.11", features = ["tls", "transport"] }
+prost = "0.12"
 hex = "0.4.3"
-bitcoin = { version = "0.30", features = [ "serde" ] }
+bitcoin = { version = "0.31", features = ["serde"] }
+tokio-stream = { version = "0.1.14", features = ["sync"] }
+tokio = { version = "1.36.0", features = ["sync"] }
+futures-core = "0.3.30"
+tokio-util = "0.7.10"
 
 [dev-dependencies]
 serde_json = "1.0.72"
-cln-rpc = { path="../cln-rpc/", version = "^0.1" }
+cln-rpc = { path = "../cln-rpc/", version = "0.4" }
 
 [build-dependencies]
-tonic-build = "0.8"
+tonic-build = "0.11"

cln-grpc/proto/primitives.proto

@@ -37,6 +37,8 @@ enum ChannelState {
 	DualopendOpenInit = 9;
 	DualopendAwaitingLockin = 10;
 	ChanneldAwaitingSplice = 11;
+	DualopendOpenCommitted = 12;
+	DualopendOpenCommittReady = 13;
 }
 
 enum HtlcState {
@@ -62,8 +64,6 @@ enum HtlcState {
 	SentRemoveAckRevocation = 19;
 }
 
-message ChannelStateChangeCause {}
-
 message Outpoint {
 	bytes txid = 1;
 	uint32 outnum = 2;
@@ -86,9 +86,9 @@ message OutputDesc {
 
 message RouteHop {
 	bytes id = 1;
-        string short_channel_id = 2;
+	string scid = 2;
 	Amount feebase = 3;
-        uint32 feeprop = 4;
+	uint32 feeprop = 4;
 	uint32 expirydelta = 5;
 }
 message Routehint {
@@ -98,6 +98,19 @@ message RoutehintList {
 	repeated Routehint hints = 2;
 }
 
+message DecodeRouteHop {
+	bytes pubkey = 1;
+	string short_channel_id = 2;
+	Amount fee_base_msat = 3;
+	uint32 fee_proportional_millionths = 4;
+	uint32 cltv_expiry_delta = 5;
+}
+message DecodeRoutehint {
+	repeated DecodeRouteHop hops = 1;
+}
+message DecodeRoutehintList {
+	repeated DecodeRoutehint hints = 2;
+}
 
 message TlvEntry {
 	uint64 type = 1;
@@ -114,4 +127,22 @@ enum ChannelTypeName {
   anchors_zero_fee_htlc_tx_even = 2;
   scid_alias_even = 3;
   zeroconf_even = 4;
+  anchors_even = 5;
+}
+
+enum AutocleanSubsystem {
+	SUCCEEDEDFORWARDS = 0;
+	FAILEDFORWARDS = 1;
+	SUCCEEDEDPAYS = 2;
+	FAILEDPAYS = 3;
+	PAIDINVOICES = 4;
+	EXPIREDINVOICES = 5;
+}
+
+enum PluginSubcommand {
+	START = 0;
+	STOP = 1;
+	RESCAN = 2;
+	STARTDIR = 3;
+	LIST = 4;
 }

cln-grpc/src/lib.rs

@@ -1,15 +1,16 @@
 // Huge json!() macros require lots of recursion
 #![recursion_limit = "1024"]
 
-#[cfg(feature = "server")]
-mod convert;
 pub mod pb;
 
-#[cfg(feature = "server")]
-mod server;
-
-#[cfg(feature = "server")]
-pub use crate::server::Server;
+cfg_if::cfg_if! {
+    if #[cfg(feature = "server")] {
+        mod convert;
+        mod server;
+        pub use server::Server;
+        pub use server::NotificationStream;
+    }
+}
 
 #[cfg(test)]
 mod test;

cln-grpc/src/pb.rs

@@ -136,7 +136,7 @@ mod convert {
         fn from(c: RouteHop) -> Self {
             Self {
                 id: cln_rpc::primitives::PublicKey::from_slice(&c.id).unwrap(),
-                scid: cln_rpc::primitives::ShortChannelId::from_str(&c.short_channel_id).unwrap(),
+                scid: cln_rpc::primitives::ShortChannelId::from_str(&c.scid).unwrap(),
                 feebase: c.feebase.unwrap().into(),
                 feeprop: c.feeprop,
                 expirydelta: c.expirydelta as u16,
@@ -167,7 +167,7 @@ mod convert {
                 feebase: Some(c.feebase.into()),
                 feeprop: c.feeprop,
                 expirydelta: c.expirydelta as u32,
-                short_channel_id: c.scid.to_string(),
+                scid: c.scid.to_string(),
             }
         }
     }
@@ -188,6 +188,65 @@ mod convert {
         }
     }
 
+    impl From<DecodeRouteHop> for cln_rpc::primitives::DecodeRoutehop {
+        fn from(c: DecodeRouteHop) -> Self {
+            Self {
+                pubkey: cln_rpc::primitives::PublicKey::from_slice(&c.pubkey).unwrap(),
+                short_channel_id: cln_rpc::primitives::ShortChannelId::from_str(
+                    &c.short_channel_id,
+                )
+                .unwrap(),
+                fee_base_msat: c.fee_base_msat.unwrap().into(),
+                fee_proportional_millionths: c.fee_proportional_millionths,
+                cltv_expiry_delta: c.cltv_expiry_delta as u16,
+            }
+        }
+    }
+
+    impl From<DecodeRoutehint> for cln_rpc::primitives::DecodeRoutehint {
+        fn from(c: DecodeRoutehint) -> Self {
+            Self {
+                hops: c.hops.into_iter().map(|h| h.into()).collect(),
+            }
+        }
+    }
+
+    impl From<DecodeRoutehintList> for cln_rpc::primitives::DecodeRoutehintList {
+        fn from(c: DecodeRoutehintList) -> Self {
+            Self {
+                hints: c.hints.into_iter().map(|h| h.into()).collect(),
+            }
+        }
+    }
+
+    impl From<cln_rpc::primitives::DecodeRoutehop> for DecodeRouteHop {
+        fn from(c: cln_rpc::primitives::DecodeRoutehop) -> Self {
+            Self {
+                pubkey: c.pubkey.serialize().to_vec(),
+                fee_base_msat: Some(c.fee_base_msat.into()),
+                fee_proportional_millionths: c.fee_proportional_millionths,
+                cltv_expiry_delta: c.cltv_expiry_delta as u32,
+                short_channel_id: c.short_channel_id.to_string(),
+            }
+        }
+    }
+
+    impl From<cln_rpc::primitives::DecodeRoutehint> for DecodeRoutehint {
+        fn from(c: cln_rpc::primitives::DecodeRoutehint) -> Self {
+            Self {
+                hops: c.hops.into_iter().map(|h| h.into()).collect(),
+            }
+        }
+    }
+
+    impl From<cln_rpc::primitives::DecodeRoutehintList> for DecodeRoutehintList {
+        fn from(c: cln_rpc::primitives::DecodeRoutehintList) -> Self {
+            Self {
+                hints: c.hints.into_iter().map(|e| e.into()).collect(),
+            }
+        }
+    }
+
     impl From<TlvStream> for cln_rpc::primitives::TlvStream {
         fn from(s: TlvStream) -> Self {
             Self {

cln-grpc/src/test.rs

@@ -276,7 +276,7 @@ fn test_keysend() {
                         "035d2b1192dfba134e10e540875d366ebc8bc353d5aa766b80c090b39c3a5d885d",
                     )
                     .unwrap(),
-                    short_channel_id: "12345x678x90".to_string(),
+                    scid: "12345x678x90".to_string(),
                     feebase: Some(Amount { msat: 123 }),
                     feeprop: 1234,
                     expirydelta: 9,
@@ -285,7 +285,7 @@ fn test_keysend() {
                         "035d2b1192dfba134e10e540875d366ebc8bc353d5aa766b80c090b39c3a5d885d",
                     )
                     .unwrap(),
-                    short_channel_id: "12345x678x90".to_string(),
+                    scid: "12345x678x90".to_string(),
                     feebase: Some(Amount { msat: 123 }),
                     feeprop: 1234,
                     expirydelta: 9,
@@ -293,6 +293,7 @@ fn test_keysend() {
                 }],
             }),
             extratlvs: None,
+            maxfee: None
         };
 
     let u: cln_rpc::model::requests::KeysendRequest = g.into();

cln-rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cln-rpc"
-version = "0.1.8"
+version = "0.4.0"
 edition = "2021"
 license = "MIT"
 description = "An async RPC client for Core Lightning."
@@ -14,7 +14,7 @@ path = "examples/getinfo.rs"
 
 [dependencies]
 anyhow = "1.0"
-bitcoin = { version = "0.30", features = [ "serde" ] }
+bitcoin = { version = "0.31", features = [ "serde" ] }
 bytes = "1"
 futures-util = { version = "0.3", features = [ "sink" ] }
 hex = "0.4.3"

cln-rpc/Makefile

@@ -2,7 +2,7 @@ cln-rpc-wrongdir:
 	$(MAKE) -C .. cln-rpc-all
 
 CLN_RPC_EXAMPLES := target/${RUST_PROFILE}/examples/cln-rpc-getinfo
-CLN_RPC_GENALL = cln-rpc/src/model.rs
+CLN_RPC_GENALL = cln-rpc/src/model.rs cln-rpc/src/notifications.rs
 CLN_RPC_SOURCES = $(shell find cln-rpc -name *.rs) ${CLN_RPC_GENALL}
 DEFAULT_TARGETS += $(CLN_RPC_EXAMPLES) $(CLN_RPC_GENALL)
 
@@ -18,4 +18,4 @@ target/${RUST_PROFILE}/examples/cln-plugin-reentrant: ${CLN_RPC_SOURCES} plugins
 	cargo build ${CARGO_OPTS} --example cln-plugin-reentrant
 
 
-cln-rpc-all: ${CLN_RPC_GEN_ALL} ${CLN_RPC_EXAMPLES}
+cln-rpc-all: ${CLN_RPC_GENALL} ${CLN_RPC_EXAMPLES}

cln-rpc/src/codec.rs

@@ -105,6 +105,7 @@ impl Decoder for JsonCodec {
 /// A codec that reads fully formed [crate::messages::JsonRpc]
 /// messages. Internally it uses the [JsonCodec] which itself is built
 /// on the [MultiLineCodec].
+#[allow(dead_code)]
 #[derive(Default)]
 pub(crate) struct JsonRpcCodec {
     inner: JsonCodec,

cln-rpc/src/lib.rs

@@ -331,6 +331,8 @@ where
 
 #[cfg(test)]
 mod test {
+    use self::notifications::{BlockAddedNotification, CustomMsgNotification};
+
     use super::*;
     use crate::model::*;
     use crate::primitives::PublicKey;
@@ -553,4 +555,64 @@ mod test {
         assert_eq!(rpc_error.code.unwrap(), 666);
         assert_eq!(rpc_error.message, "MOCK_ERROR");
     }
+
+    #[test]
+    fn serialize_custom_msg_notification() {
+        let msg = CustomMsgNotification {
+            peer_id : PublicKey::from_str("0364aeb75519be29d1af7b8cc6232dbda9fdabb79b66e4e1f6a223750954db210b").unwrap(),
+            payload : String::from("941746573749")
+        };
+
+        let notification = Notification::CustomMsg(msg);
+
+        assert_eq!(
+            serde_json::to_value(notification).unwrap(),
+            serde_json::json!(
+                {
+                    "custommsg" : {
+                        "peer_id" : "0364aeb75519be29d1af7b8cc6232dbda9fdabb79b66e4e1f6a223750954db210b",
+                        "payload" : "941746573749"
+                    }
+                }
+            )
+        );
+
+    }
+
+    #[test]
+    fn serialize_block_added_notification() {
+        let block_added = BlockAddedNotification {
+            hash : crate::primitives::Sha256::from_str("000000000000000000000acab8abe0c67a52ed7e5a90a19c64930ff11fa84eca").unwrap(),
+            height : 830702
+        };
+
+        let notification = Notification::BlockAdded(block_added);
+
+        assert_eq!(
+            serde_json::to_value(notification).unwrap(),
+            serde_json::json!({
+                "block_added" : {
+                    "hash" : "000000000000000000000acab8abe0c67a52ed7e5a90a19c64930ff11fa84eca",
+                    "height" : 830702
+                }
+            })
+        )
+    }
+
+    #[test]
+    fn deserialize_connect_notification() {
+        let connect_json = serde_json::json!({
+            "connect" :  {
+                "address" : {
+                    "address" : "127.0.0.1",
+                    "port" : 38012,
+                    "type" : "ipv4"
+                },
+                "direction" : "in",
+                "id" : "022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59"
+            }
+        });
+
+        let _ : Notification = serde_json::from_value(connect_json).unwrap();
+    }
 }

cln-rpc/src/notifications.rs

@@ -1,4 +1,231 @@
-use serde::{Deserialize, Serialize};
+// This file is autogenerated by `msggen`
+// Do not edit it manually, your changes will be overwritten
 
-#[derive(Debug, Deserialize, Serialize)]
-pub enum Notification {}
+
+
+use crate::primitives::*;
+use serde::{Serialize, Deserialize};
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub enum Notification {
+    #[serde(rename = "block_added")]
+    BlockAdded(BlockAddedNotification),
+    #[serde(rename = "channel_open_failed")]
+    ChannelOpenFailed(ChannelOpenFailedNotification),
+    #[serde(rename = "channel_opened")]
+    ChannelOpened(ChannelOpenedNotification),
+    #[serde(rename = "connect")]
+    Connect(ConnectNotification),
+    #[serde(rename = "custommsg")]
+    CustomMsg(CustomMsgNotification),
+    #[serde(rename = "channel_state_changed")]
+    ChannelStateChanged(ChannelStateChangedNotification),
+}
+
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct BlockAddedNotification {
+    pub hash: Sha256,
+    pub height: u32,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ChannelOpenFailedNotification {
+    pub channel_id: Sha256,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ChannelOpenedNotification {
+    pub channel_ready: bool,
+    pub funding_msat: Amount,
+    pub funding_txid: String,
+    pub id: PublicKey,
+}
+
+/// ['Direction of the connection']
+#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
+#[allow(non_camel_case_types)]
+pub enum ConnectDirection {
+    #[serde(rename = "in")]
+    IN = 0,
+    #[serde(rename = "out")]
+    OUT = 1,
+}
+
+impl TryFrom<i32> for ConnectDirection {
+    type Error = anyhow::Error;
+    fn try_from(c: i32) -> Result<ConnectDirection, anyhow::Error> {
+        match c {
+    0 => Ok(ConnectDirection::IN),
+    1 => Ok(ConnectDirection::OUT),
+            o => Err(anyhow::anyhow!("Unknown variant {} for enum ConnectDirection", o)),
+        }
+    }
+}
+
+impl ToString for ConnectDirection {
+    fn to_string(&self) -> String {
+        match self {
+            ConnectDirection::IN => "IN",
+            ConnectDirection::OUT => "OUT",
+        }.to_string()
+    }
+}
+
+/// ['Type of connection (*torv2*/*torv3* only if **direction** is *out*)']
+#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
+#[allow(non_camel_case_types)]
+pub enum ConnectAddressType {
+    #[serde(rename = "local socket")]
+    LOCAL_SOCKET = 0,
+    #[serde(rename = "ipv4")]
+    IPV4 = 1,
+    #[serde(rename = "ipv6")]
+    IPV6 = 2,
+    #[serde(rename = "torv2")]
+    TORV2 = 3,
+    #[serde(rename = "torv3")]
+    TORV3 = 4,
+}
+
+impl TryFrom<i32> for ConnectAddressType {
+    type Error = anyhow::Error;
+    fn try_from(c: i32) -> Result<ConnectAddressType, anyhow::Error> {
+        match c {
+    0 => Ok(ConnectAddressType::LOCAL_SOCKET),
+    1 => Ok(ConnectAddressType::IPV4),
+    2 => Ok(ConnectAddressType::IPV6),
+    3 => Ok(ConnectAddressType::TORV2),
+    4 => Ok(ConnectAddressType::TORV3),
+            o => Err(anyhow::anyhow!("Unknown variant {} for enum ConnectAddressType", o)),
+        }
+    }
+}
+
+impl ToString for ConnectAddressType {
+    fn to_string(&self) -> String {
+        match self {
+            ConnectAddressType::LOCAL_SOCKET => "LOCAL_SOCKET",
+            ConnectAddressType::IPV4 => "IPV4",
+            ConnectAddressType::IPV6 => "IPV6",
+            ConnectAddressType::TORV2 => "TORV2",
+            ConnectAddressType::TORV3 => "TORV3",
+        }.to_string()
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ConnectAddress {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub address: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub port: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub socket: Option<String>,
+    // Path `connect.address.type`
+    #[serde(rename = "type")]
+    pub item_type: ConnectAddressType,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ConnectNotification {
+    // Path `connect.direction`
+    pub direction: ConnectDirection,
+    pub address: ConnectAddress,
+    pub id: PublicKey,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct CustomMsgNotification {
+    pub payload: String,
+    pub peer_id: PublicKey,
+}
+
+/// ['The cause of the state change.']
+#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
+#[allow(non_camel_case_types)]
+pub enum ChannelStateChangedCause {
+    #[serde(rename = "unknown")]
+    UNKNOWN = 0,
+    #[serde(rename = "local")]
+    LOCAL = 1,
+    #[serde(rename = "user")]
+    USER = 2,
+    #[serde(rename = "remote")]
+    REMOTE = 3,
+    #[serde(rename = "protocol")]
+    PROTOCOL = 4,
+    #[serde(rename = "onchain")]
+    ONCHAIN = 5,
+}
+
+impl TryFrom<i32> for ChannelStateChangedCause {
+    type Error = anyhow::Error;
+    fn try_from(c: i32) -> Result<ChannelStateChangedCause, anyhow::Error> {
+        match c {
+    0 => Ok(ChannelStateChangedCause::UNKNOWN),
+    1 => Ok(ChannelStateChangedCause::LOCAL),
+    2 => Ok(ChannelStateChangedCause::USER),
+    3 => Ok(ChannelStateChangedCause::REMOTE),
+    4 => Ok(ChannelStateChangedCause::PROTOCOL),
+    5 => Ok(ChannelStateChangedCause::ONCHAIN),
+            o => Err(anyhow::anyhow!("Unknown variant {} for enum ChannelStateChangedCause", o)),
+        }
+    }
+}
+
+impl ToString for ChannelStateChangedCause {
+    fn to_string(&self) -> String {
+        match self {
+            ChannelStateChangedCause::UNKNOWN => "UNKNOWN",
+            ChannelStateChangedCause::LOCAL => "LOCAL",
+            ChannelStateChangedCause::USER => "USER",
+            ChannelStateChangedCause::REMOTE => "REMOTE",
+            ChannelStateChangedCause::PROTOCOL => "PROTOCOL",
+            ChannelStateChangedCause::ONCHAIN => "ONCHAIN",
+        }.to_string()
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ChannelStateChangedNotification {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub old_state: Option<ChannelState>,
+    // Path `channel_state_changed.cause`
+    pub cause: ChannelStateChangedCause,
+    // Path `channel_state_changed.new_state`
+    pub new_state: ChannelState,
+    pub channel_id: Sha256,
+    pub message: String,
+    pub peer_id: PublicKey,
+    pub short_channel_id: ShortChannelId,
+    pub timestamp: String,
+}
+
+pub mod requests{
+use serde::{Serialize, Deserialize};
+
+    #[derive(Clone, Debug, Deserialize, Serialize)]
+    pub struct StreamBlockAddedRequest {
+    }
+
+    #[derive(Clone, Debug, Deserialize, Serialize)]
+    pub struct StreamChannelOpenFailedRequest {
+    }
+
+    #[derive(Clone, Debug, Deserialize, Serialize)]
+    pub struct StreamChannelOpenedRequest {
+    }
+
+    #[derive(Clone, Debug, Deserialize, Serialize)]
+    pub struct StreamConnectRequest {
+    }
+
+    #[derive(Clone, Debug, Deserialize, Serialize)]
+    pub struct StreamCustomMsgRequest {
+    }
+
+    #[derive(Clone, Debug, Deserialize, Serialize)]
+    pub struct StreamChannelStateChangedRequest {
+    }
+
+}

cln-rpc/src/primitives.rs

@@ -12,7 +12,7 @@ use std::string::ToString;
 pub use bitcoin::hashes::sha256::Hash as Sha256;
 pub use bitcoin::secp256k1::PublicKey;
 
-#[derive(Copy, Clone, Serialize, Deserialize, Debug, PartialEq)]
+#[derive(Copy, Clone, Serialize, Deserialize, Debug, PartialEq, Eq)]
 #[allow(non_camel_case_types)]
 pub enum ChannelState {
     OPENINGD = 0,
@@ -27,9 +27,11 @@ pub enum ChannelState {
     DUALOPEND_OPEN_INIT = 9,
     DUALOPEND_AWAITING_LOCKIN = 10,
     CHANNELD_AWAITING_SPLICE = 11,
+    DUALOPEND_OPEN_COMMITTED = 12,
+    DUALOPEND_OPEN_COMMIT_READY = 13,
 }
 
-#[derive(Copy, Clone, Serialize, Deserialize, Debug)]
+#[derive(Copy, Clone, Serialize, Deserialize, Debug, PartialEq, Eq)]
 #[allow(non_camel_case_types)]
 pub enum HtlcState {
     SENT_ADD_HTLC = 0,
@@ -67,18 +69,71 @@ pub enum ChannelTypeName {
     SCID_ALIAS_EVEN = 3,
     #[serde(rename = "zeroconf/even")]
     ZEROCONF_EVEN = 4,
+    #[serde(rename = "anchors/even")]
+    ANCHORS_EVEN = 5,
 }
 
 #[derive(Copy, Clone, Serialize, Deserialize, Debug)]
 #[allow(non_camel_case_types)]
-#[serde(rename_all = "lowercase")]
-pub enum ChannelStateChangeCause {
-    UNKNOWN,
-    LOCAL,
-    USER,
-    REMOTE,
-    PROTOCOL,
-    ONCHAIN,
+pub enum AutocleanSubsystem {
+    #[serde(rename = "succeededforwards")]
+    SUCCEEDEDFORWARDS = 0,
+    #[serde(rename = "failedforwards")]
+    FAILEDFORWARDS = 1,
+    #[serde(rename = "succeededpays")]
+    SUCCEEDEDPAYS = 2,
+    #[serde(rename = "failedpays")]
+    FAILEDPAYS = 3,
+    #[serde(rename = "paidinvoices")]
+    PAIDINVOICES = 4,
+    #[serde(rename = "expiredinvoices")]
+    EXPIREDINVOICES = 5,
+}
+
+impl TryFrom<i32> for AutocleanSubsystem {
+    type Error = crate::Error;
+
+    fn try_from(value: i32) -> std::result::Result<Self, Self::Error> {
+        match value {
+            0 => Ok(AutocleanSubsystem::SUCCEEDEDFORWARDS),
+            1 => Ok(AutocleanSubsystem::FAILEDFORWARDS),
+            2 => Ok(AutocleanSubsystem::SUCCEEDEDPAYS),
+            3 => Ok(AutocleanSubsystem::FAILEDPAYS),
+            4 => Ok(AutocleanSubsystem::PAIDINVOICES),
+            5 => Ok(AutocleanSubsystem::EXPIREDINVOICES),
+            _ => Err(anyhow!("Invalid AutocleanSubsystem {}", value)),
+        }
+    }
+}
+
+#[derive(Copy, Clone, Serialize, Deserialize, Debug)]
+#[allow(non_camel_case_types)]
+pub enum PluginSubcommand {
+    #[serde(rename = "start")]
+    START = 0,
+    #[serde(rename = "stop")]
+    STOP = 1,
+    #[serde(rename = "rescan")]
+    RESCAN = 2,
+    #[serde(rename = "startdir")]
+    STARTDIR = 3,
+    #[serde(rename = "list")]
+    LIST = 4,
+}
+
+impl TryFrom<i32> for PluginSubcommand {
+    type Error = crate::Error;
+
+    fn try_from(value: i32) -> std::result::Result<Self, Self::Error> {
+        match value {
+            0 => Ok(PluginSubcommand::START),
+            1 => Ok(PluginSubcommand::STOP),
+            2 => Ok(PluginSubcommand::RESCAN),
+            3 => Ok(PluginSubcommand::STARTDIR),
+            4 => Ok(PluginSubcommand::LIST),
+            _ => Err(anyhow!("Invalid PluginSubcommand mapping!")),
+        }
+    }
 }
 
 /// An `Amount` that can also be `any`. Useful for cases in which you
@@ -182,6 +237,11 @@ impl FromStr for ShortChannelId {
         ))
     }
 }
+impl From<u64> for ShortChannelId {
+    fn from(scid: u64) -> Self {
+        ShortChannelId(scid)
+    }
+}
 impl Display for ShortChannelId {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}x{}x{}", self.block(), self.txindex(), self.outnum())
@@ -197,6 +257,63 @@ impl ShortChannelId {
     pub fn outnum(&self) -> u16 {
         self.0 as u16 & 0xFFFF
     }
+    pub fn to_u64(&self) -> u64 {
+        self.0
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct ShortChannelIdDir {
+    pub short_channel_id: ShortChannelId,
+    pub direction: u32,
+}
+impl Serialize for ShortChannelIdDir {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&self.to_string())
+    }
+}
+
+impl<'de> Deserialize<'de> for ShortChannelIdDir {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        use serde::de::Error;
+        let s: String = Deserialize::deserialize(deserializer)?;
+        Ok(Self::from_str(&s).map_err(|e| Error::custom(e.to_string()))?)
+    }
+}
+impl FromStr for ShortChannelIdDir {
+    type Err = crate::Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let parts: Result<Vec<String>, _> = s.split('/').map(|p| p.parse()).collect();
+        let parts = parts.with_context(|| format!("Malformed short_channel_id_dir: {}", s))?;
+        if parts.len() != 2 {
+            return Err(anyhow!(
+                "Malformed short_channel_id_dir: element count mismatch"
+            ));
+        }
+
+        Ok(ShortChannelIdDir {
+            short_channel_id: ShortChannelId::from_str(&parts[0])?,
+            direction: parts[1].parse::<u32>()?,
+        })
+    }
+}
+impl Display for ShortChannelIdDir {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}x{}x{}/{}",
+            self.short_channel_id.block(),
+            self.short_channel_id.txindex(),
+            self.short_channel_id.outnum(),
+            self.direction
+        )
+    }
 }
 
 #[derive(Clone, Copy, Debug)]
@@ -290,7 +407,7 @@ impl<'de> Deserialize<'de> for Outpoint {
     }
 }
 
-#[derive(Copy, Clone, Serialize, Deserialize, Debug, PartialEq)]
+#[derive(Copy, Clone, Serialize, Deserialize, Debug, PartialEq, Eq)]
 #[serde(rename_all = "lowercase")]
 pub enum ChannelSide {
     LOCAL,
@@ -328,6 +445,8 @@ impl TryFrom<i32> for ChannelState {
             9 => Ok(ChannelState::DUALOPEND_OPEN_INIT),
             10 => Ok(ChannelState::DUALOPEND_AWAITING_LOCKIN),
             11 => Ok(ChannelState::CHANNELD_AWAITING_SPLICE),
+            12 => Ok(ChannelState::DUALOPEND_OPEN_COMMITTED),
+            13 => Ok(ChannelState::DUALOPEND_OPEN_COMMIT_READY),
             _ => Err(anyhow!("Invalid channel state {}", value)),
         }
     }
@@ -341,6 +460,7 @@ impl From<i32> for ChannelTypeName {
             2 => ChannelTypeName::ANCHORS_ZERO_FEE_HTLC_TX_EVEN,
             3 => ChannelTypeName::SCID_ALIAS_EVEN,
             4 => ChannelTypeName::ZEROCONF_EVEN,
+            5 => ChannelTypeName::ANCHORS_EVEN,
             o => panic!("Unmapped ChannelTypeName {}", o),
         }
     }
@@ -354,6 +474,7 @@ impl From<ChannelTypeName> for i32 {
             ChannelTypeName::ANCHORS_ZERO_FEE_HTLC_TX_EVEN => 2,
             ChannelTypeName::SCID_ALIAS_EVEN => 3,
             ChannelTypeName::ZEROCONF_EVEN => 4,
+            ChannelTypeName::ANCHORS_EVEN => 5,
         }
     }
 }
@@ -730,7 +851,7 @@ mod test {
         });
 
         let p: FundchannelResponse = serde_json::from_value(r).unwrap();
-	assert_eq!(p.channel_type.unwrap().bits, Some(vec![1,3,5]));
+        assert_eq!(p.channel_type.unwrap().bits, vec![1, 3, 5]);
     }
 }
 
@@ -818,20 +939,91 @@ impl Serialize for RoutehintList {
 }
 
 impl<'de> Deserialize<'de> for RoutehintList {
-    fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: Deserializer<'de>,
     {
-        todo!("Required once we roundtrip, but not necessary for cln-rpc itself")
+        let hints: Vec<Routehint> = Vec::deserialize(deserializer)?;
+
+        Ok(RoutehintList { hints })
     }
 }
 
 impl<'de> Deserialize<'de> for Routehint {
-    fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: Deserializer<'de>,
     {
-        todo!("Required once we roundtrip, but not necessary for cln-rpc itself")
+        let hops: Vec<Routehop> = Vec::deserialize(deserializer)?;
+
+        Ok(Routehint { hops })
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct DecodeRoutehop {
+    pub pubkey: PublicKey,
+    pub short_channel_id: ShortChannelId,
+    pub fee_base_msat: Amount,
+    pub fee_proportional_millionths: u32,
+    pub cltv_expiry_delta: u16,
+}
+
+#[derive(Clone, Debug)]
+pub struct DecodeRoutehint {
+    pub hops: Vec<DecodeRoutehop>,
+}
+
+#[derive(Clone, Debug)]
+pub struct DecodeRoutehintList {
+    pub hints: Vec<DecodeRoutehint>,
+}
+
+impl Serialize for DecodeRoutehint {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut seq = serializer.serialize_seq(Some(self.hops.len()))?;
+        for e in self.hops.iter() {
+            seq.serialize_element(e)?;
+        }
+        seq.end()
+    }
+}
+
+impl Serialize for DecodeRoutehintList {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut seq = serializer.serialize_seq(Some(self.hints.len()))?;
+        for e in self.hints.iter() {
+            seq.serialize_element(e)?;
+        }
+        seq.end()
+    }
+}
+
+impl<'de> Deserialize<'de> for DecodeRoutehintList {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let hints: Vec<DecodeRoutehint> = Vec::deserialize(deserializer)?;
+
+        Ok(DecodeRoutehintList { hints })
+    }
+}
+
+impl<'de> Deserialize<'de> for DecodeRoutehint {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let hops: Vec<DecodeRoutehop> = Vec::deserialize(deserializer)?;
+
+        Ok(DecodeRoutehint { hops })
     }
 }
 

closingd/Makefile

@@ -57,9 +57,7 @@ CLOSINGD_COMMON_OBJS :=				\
 	common/utxo.o				\
 	common/version.o			\
 	common/wire_error.o			\
-	common/wireaddr.o			\
-	gossipd/gossip_store_wiregen.o		\
-	gossipd/gossipd_peerd_wiregen.o
+	common/wireaddr.o
 
 lightningd/lightning_closingd: $(CLOSINGD_OBJS) $(CLOSINGD_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(HSMD_CLIENT_OBJS)
 

closingd/closingd.c

@@ -20,7 +20,6 @@
 #include <common/version.h>
 #include <common/wire_error.h>
 #include <errno.h>
-#include <gossipd/gossipd_peerd_wiregen.h>
 #include <hsmd/hsmd_wiregen.h>
 #include <inttypes.h>
 #include <stdio.h>
@@ -128,7 +127,7 @@ static u8 *closing_read_peer_msg(const tal_t *ctx,
 static void send_offer(struct per_peer_state *pps,
 		       const struct chainparams *chainparams,
 		       const struct channel_id *channel_id,
-		       const struct pubkey funding_pubkey[NUM_SIDES],
+		       const struct pubkey *funding_pubkey,
 		       const u8 *funding_wscript,
 		       u32 *local_wallet_index,
 		       const struct ext_key *local_wallet_ext_key,
@@ -166,7 +165,7 @@ static void send_offer(struct per_peer_state *pps,
 
 	/* BOLT #3:
 	 *
-	 * ## Closing Transaction
+	 * ## Legacy Closing Transaction
 	 *...
 	 * Each node offering a signature... MAY eliminate its
 	 * own output.
@@ -227,7 +226,7 @@ static struct amount_sat
 receive_offer(struct per_peer_state *pps,
 	      const struct chainparams *chainparams,
 	      const struct channel_id *channel_id,
-	      const struct pubkey funding_pubkey[NUM_SIDES],
+	      const struct pubkey *funding_pubkey,
 	      const u8 *funding_wscript,
 	      u32 *local_wallet_index,
 	      const struct ext_key *local_wallet_ext_key,
@@ -268,8 +267,8 @@ receive_offer(struct per_peer_state *pps,
 		 */
 		else if (fromwire_peektype(msg) == WIRE_SHUTDOWN)
 			msg = tal_free(msg);
-		/* channeld may have sent ping: ignore pong! */
-		else if (fromwire_peektype(msg) == WIRE_PONG)
+		/* We can get announcement signatures: too late! */
+		else if (fromwire_peektype(msg) == WIRE_ANNOUNCEMENT_SIGNATURES)
 			msg = tal_free(msg);
 	} while (!msg);
 

common/Makefile

@@ -16,6 +16,7 @@ COMMON_SRC_NOGEN :=				\
 	common/bolt11.c				\
 	common/bolt11_json.c			\
 	common/bolt12.c				\
+	common/bolt12_id.c			\
 	common/bolt12_merkle.c			\
 	common/channel_config.c			\
 	common/channel_id.c			\
@@ -52,9 +53,10 @@ COMMON_SRC_NOGEN :=				\
 	common/interactivetx.c			\
 	common/initial_channel.c		\
 	common/initial_commit_tx.c		\
-	common/invoice_path_id.c		\
 	common/iso4217.c			\
+	common/json_blinded_path.c		\
 	common/json_channel_type.c		\
+	common/json_command.c			\
 	common/json_filter.c			\
 	common/json_param.c			\
 	common/json_parse.c			\
@@ -69,6 +71,7 @@ COMMON_SRC_NOGEN :=				\
 	common/onion_decode.c			\
 	common/onion_encode.c			\
 	common/onionreply.c			\
+	common/onion_message.c			\
 	common/onion_message_parse.c		\
 	common/peer_billboard.c			\
 	common/peer_failed.c			\
@@ -77,6 +80,7 @@ COMMON_SRC_NOGEN :=				\
 	common/per_peer_state.c			\
 	common/permute_tx.c			\
 	common/ping.c				\
+	common/plugin.c				\
 	common/psbt_internal.c			\
 	common/psbt_keypath.c			\
 	common/psbt_open.c			\
@@ -84,9 +88,11 @@ COMMON_SRC_NOGEN :=				\
 	common/random_select.c			\
 	common/read_peer_msg.c			\
 	common/route.c				\
+	common/sciddir_or_pubkey.c		\
 	common/setup.c				\
 	common/shutdown_scriptpubkey.c		\
 	common/sphinx.c				\
+	common/splice_script.c			\
 	common/status.c				\
 	common/status_levels.c			\
 	common/status_wire.c			\
@@ -112,7 +118,6 @@ COMMON_HEADERS_NOGEN := $(COMMON_SRC_NOGEN:.c=.h)	\
 	common/gossip_constants.h			\
 	common/hsm_version.h				\
 	common/htlc.h					\
-	common/json_command.h				\
 	common/jsonrpc_errors.h				\
 	common/overflows.h
 

common/addr.c

@@ -36,3 +36,79 @@ char *encode_scriptpubkey_to_addr(const tal_t *ctx,
 
 	return out;
 }
+
+static const char *segwit_addr_net_decode(int *witness_version,
+					  uint8_t *witness_program,
+					  size_t *witness_program_len,
+					  const char *addrz,
+					  const struct chainparams *chainparams)
+{
+	if (segwit_addr_decode(witness_version, witness_program,
+			       witness_program_len, chainparams->onchain_hrp,
+			       addrz))
+		return chainparams->onchain_hrp;
+	else
+		return NULL;
+}
+
+bool decode_scriptpubkey_from_addr(const tal_t *ctx,
+				   const struct chainparams *chainparams,
+				   const char *address,
+				   u8 **scriptpubkey)
+{
+	struct bitcoin_address destination;
+	int witness_version;
+	/* segwit_addr_net_decode requires a buffer of size 40, and will
+	 * not write to the buffer if the address is too long, so a buffer
+	 * of fixed size 40 will not overflow. */
+	uint8_t witness_program[40];
+	size_t witness_program_len;
+	const char *bech32;
+	u8 addr_version;
+
+	if (ripemd160_from_base58(&addr_version, &destination.addr,
+				  address, strlen(address))) {
+		if (addr_version == chainparams->p2pkh_version) {
+			*scriptpubkey = scriptpubkey_p2pkh(ctx, &destination);
+			return true;
+		} else if (addr_version == chainparams->p2sh_version) {
+			*scriptpubkey =
+			    scriptpubkey_p2sh_hash(ctx, &destination.addr);
+			return true;
+		} else {
+			return false;
+		}
+		/* Insert other parsers that accept pointer+len here. */
+		return false;
+	}
+
+	bech32 = segwit_addr_net_decode(&witness_version, witness_program,
+					&witness_program_len, address,
+					chainparams);
+	if (bech32) {
+		bool witness_ok;
+
+		if (witness_version == 0) {
+			witness_ok = (witness_program_len == 20 ||
+				       witness_program_len == 32);
+		} else if (witness_version == 1) {
+			witness_ok = (witness_program_len == 32);
+		} else {
+			witness_ok = true;
+		}
+
+		if (!witness_ok)
+			return false;
+
+		if (!streq(bech32, chainparams->onchain_hrp))
+			return false;
+
+		*scriptpubkey = scriptpubkey_witness_raw(ctx, witness_version,
+							 witness_program,
+							 witness_program_len);
+		return true;
+	}
+
+	/* Insert other parsers that accept null-terminated string here. */
+	return false;
+}