Dockerfiles must not invalidate the docker cache

.dockerignore

@@ -1,3 +1,2 @@
 Dockerfile
 contrib/docker/Dockerfile.*
-target

.gitattributes

@@ -17,4 +17,4 @@ statements_gettextgen.po linguist-generated=true
 cln-grpc/proto/node.proto -text -diff linguist-generated=true
 cln-grpc/src/convert.rs -text -diff linguist-generated=true
 cln-rpc/src/model.rs -text -diff linguist-generated=true
-contrib/pyln-testing/pyln/testing/node_pb2.py -text -diff linguist-generated=true
+contrib/pyln-testing/pyln/testing/node_pb2.py linguist-generated=true

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,16 +0,0 @@
-> [!IMPORTANT]
->
-> 25.09 FREEZE July 28TH: Non-bugfix PRs not ready by this date will wait for 25.12.
->
-> RC1 is scheduled on _August 11th_
->
-> The final release is scheduled for September 1st.
-
-
-## Checklist
-Before submitting the PR, ensure the following tasks are completed. If an item is not applicable to your PR, please mark it as checked:
-
-- [ ] The changelog has been updated in the relevant commit(s) according to the [guidelines](https://docs.corelightning.org/docs/coding-style-guidelines#changelog-entries-in-commit-messages).
-- [ ] Tests have been added or modified to reflect the changes.
-- [ ] Documentation has been reviewed and updated as needed.
-- [ ] Related issues have been listed and linked, including any that this PR closes.

.github/scripts/install-bitcoind.sh

@@ -2,30 +2,18 @@
 
 set -e
 
-export BITCOIN_VERSION=27.1
-export ELEMENTS_VERSION=23.2.1
-
 DIRNAME="bitcoin-${BITCOIN_VERSION}"
 EDIRNAME="elements-${ELEMENTS_VERSION}"
 FILENAME="${DIRNAME}-x86_64-linux-gnu.tar.gz"
 EFILENAME="${EDIRNAME}-x86_64-linux-gnu.tar.gz"
 
 cd /tmp/
+wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/${FILENAME}"
+wget "https://github.com/ElementsProject/elements/releases/download/elements-${ELEMENTS_VERSION}/${EFILENAME}"
+tar -xf "${FILENAME}"
+tar -xf "${EFILENAME}"
+sudo mv "${DIRNAME}"/bin/* "/usr/local/bin"
+sudo mv "${EDIRNAME}"/bin/* "/usr/local/bin"
 
-# Since we inadvertently broke `elementsd` support in the past we only
-# want to download and enable the daemon that is actually going to be
-# used when running in CI. Otherwise we could end up accidentally
-# testing against `bitcoind` but still believe that we ran against
-# `elementsd`.
-if [ "$TEST_NETWORK" = "liquid-regtest" ]; then
-    wget "https://github.com/ElementsProject/elements/releases/download/elements-${ELEMENTS_VERSION}/${EFILENAME}"
-    tar -xf "${EFILENAME}"
-    sudo mv "${EDIRNAME}"/bin/* "/usr/local/bin"
-    rm -rf "${EFILENAME}" "${EDIRNAME}"
-else
-    wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/${FILENAME}"
-    tar -xf "${FILENAME}"
-    sudo mv "${DIRNAME}"/bin/* "/usr/local/bin"
-    rm -rf "${FILENAME}" "${DIRNAME}"
-fi
 
+rm -rf "${FILENAME}" "${EFILENAME}" "${DIRNAME}" "${EDIRNAME}"

.github/scripts/setup.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 export DEBIAN_FRONTEND=noninteractive
+export BITCOIN_VERSION=25.0
+export ELEMENTS_VERSION=22.0.2
 export RUST_VERSION=stable
 
 sudo useradd -ms /bin/bash tester
@@ -20,18 +22,11 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      gcc-arm-none-eabi \
      gettext \
      git \
-     gnupg \
-     jq \
      libc6-dev-arm64-cross \
      libc6-dev-armhf-cross \
-     libev-dev \
-     libevent-dev \
-     libffi-dev \
-     libicu-dev \
      libpq-dev \
      libprotobuf-c-dev \
      libsqlite3-dev \
-     libssl-dev \
      libtool \
      libxml2-utils \
      locales \
@@ -49,18 +44,29 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      software-properties-common \
      sudo \
      tcl \
-     tclsh \
      unzip \
      valgrind \
      wget \
      xsltproc \
-     systemtap-sdt-dev \
      zlib1g-dev
 
 echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
 sudo chmod 0440 /etc/sudoers.d/tester
 
-"$(dirname "$0")"/install-bitcoind.sh
+(
+    cd /tmp/ || exit 1
+    wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
+    wget https://github.com/ElementsProject/elements/releases/download/elements-${ELEMENTS_VERSION}/elements-${ELEMENTS_VERSION}-x86_64-linux-gnu.tar.gz
+    tar -xf bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
+    tar -xf elements-${ELEMENTS_VERSION}-x86_64-linux-gnu.tar.gz
+    sudo mv bitcoin-${BITCOIN_VERSION}/bin/* /usr/local/bin
+    sudo mv elements-${ELEMENTS_VERSION}/bin/* /usr/local/bin
+    rm -rf \
+       bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz \
+       bitcoin-${BITCOIN_VERSION} \
+       elements-${ELEMENTS_VERSION}-x86_64-linux-gnu.tar.gz \
+       elements-${ELEMENTS_VERSION}
+)
 
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
      -y --default-toolchain ${RUST_VERSION}
@@ -72,17 +78,20 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
 # which comes from the same tree.  Makes sense!
 
 # And
-#   grpcio-tools-1.69.0` requires `protobuf = ">=5.26.1,<6.0dev"`
+#   grpcio-tools-1.54.0` requires `protobuf = ">=4.21.6,<5.0dev"`
 
 # Now, protoc changed to date-based releases, BUT Python protobuf
 # didn't, so Python protobuf 4.21.12 (in Ubuntu 23.04) corresponds to
 # protoc 21.12 (which, FYI, is packaged in Ubuntu as version 3.21.12).
 
-# In general protobuf version x.y.z corresponds to protoc version y.z
+# So we're going to nail these versions as 21.12, which is what recent
+# Ubuntu has, and hopefully everyone else can get.  And this means that
+# When CI checks that no files have changed under regeneration, you won't
+# get a fail just because the dev's protoc is a different version.
 
 # Honorable mention go to Matt Whitlock for spelunking this horror with me!
 
-PROTOC_VERSION=29.4
+PROTOC_VERSION=21.12
 PB_REL="https://github.com/protocolbuffers/protobuf/releases"
 curl -LO $PB_REL/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip
 sudo unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local/

.github/scripts/sync-rpc-cmds.py

@@ -2,30 +2,26 @@ import os
 from time import sleep
 import requests
 import re
-from enum import Enum
 
 # readme url
-URL = "https://dash.readme.com/api/v1"
+URL = "https://dash.readme.com/api/v1/docs"
 # category id for API reference
 CATEGORY_ID = "63e4e160c60b2e001dd1cc4e"
-CATEGORY_SLUG = "json-rpc-apis"
 
 
-class Action(Enum):
-    ADD = 'add'
-    UPDATE = 'update'
-    DELETE = 'delete'
+def checkIfDocIsPresent(title, headers):
 
+    check_url = URL + "/" + title
+    response = requests.get(check_url, headers=headers)
 
-def getListOfRPCDocs(headers):
-    response = requests.get(f"{URL}/categories/{CATEGORY_SLUG}/docs", headers=headers)
     if response.status_code == 200:
-        return response.json()
+        return True
     else:
-        return []
+        return False
 
 
-def publishDoc(action, title, body, order, headers):
+def publishDoc(title, body, order):
+    key = os.environ.get("README_API_KEY")
     payload = {
         "title": title,
         "type": "basic",
@@ -34,30 +30,28 @@ def publishDoc(action, title, body, order, headers):
         "hidden": False,
         "order": order,
     }
-    # title == slug
-    if action == Action.ADD:
-        # create doc
-        response = requests.post(URL + "/docs", json=payload, headers=headers)
-        if response.status_code != 201:
-            print(response.text)
-        else:
-            print("Created ", title)
-    elif action == Action.UPDATE:
+    headers = {
+        "accept": "application/json",
+        "content-type": "application/json",
+        "authorization": "Basic " + key,
+    }
+
+    isDocPresent = checkIfDocIsPresent(title, headers)
+    if isDocPresent:
         # update doc
-        response = requests.put(f"{URL}/docs/{title}", json=payload, headers=headers)
+        update_url = URL + "/" + title  # title == slug
+        response = requests.put(update_url, json=payload, headers=headers)
         if response.status_code != 200:
             print(response.text)
         else:
             print("Updated ", title)
-    elif action == Action.DELETE:
-        # delete doc
-        response = requests.delete(f"{URL}/docs/{title}", headers=headers)
-        if response.status_code != 204:
+    else:
+        # create doc
+        response = requests.post(URL, json=payload, headers=headers)
+        if response.status_code != 201:
             print(response.text)
         else:
-            print("Deleted ", title)
-    else:
-        print("Invalid action")
+            print("Created ", title)
 
 
 def extract_rpc_commands(rst_content):
@@ -75,36 +69,19 @@ def extract_rpc_commands(rst_content):
 
 
 def main():
-    # define headers for requests
-    headers = {
-        "accept": "application/json",
-        "content-type": "application/json",
-        "authorization": "Basic " + os.environ.get("README_API_KEY"),
-    }
-
     # path to the rst file from where we fetch all the RPC commands
     path_to_rst = "doc/index.rst"
     with open(path_to_rst, "r") as file:
         rst_content = file.read()
 
-    commands_from_local = extract_rpc_commands(rst_content)
-    commands_from_readme = getListOfRPCDocs(headers)
-
-    # Compare local and server commands list to get the list of command to add or delete
-    commands_local_title = set(command[0] for command in commands_from_local)
-    commands_readme_title = set(command['title'] for command in commands_from_readme)
-    commands_to_delete = commands_readme_title - commands_local_title
-    commands_to_add = commands_local_title - commands_readme_title
-    for name in commands_to_delete:
-        publishDoc(Action.DELETE, name, "", 0, headers)
-        sleep(3)
-
-    if commands_from_local:
+    commands = extract_rpc_commands(rst_content)
+    if commands:
         order = 0
-        for name, file in commands_from_local:
+        for name, file in commands:
+            print(f"{name}\t\t{file}")
             with open("doc/" + file) as f:
                 body = f.read()
-                publishDoc(Action.ADD if name in commands_to_add else Action.UPDATE, name, body, order, headers)
+                publishDoc(name, body, order)
             order = order + 1
             sleep(3)
     else:

.github/workflows/bsd.yml

@@ -5,35 +5,28 @@ on:
     branches:
       - "master"
   pull_request:
-  workflow_dispatch:
 
 jobs:
   testfreebsd:
-    runs-on: ubuntu-22.04
+    runs-on: macos-10.15
     name: Build and test on FreeBSD
     timeout-minutes: 120
-    strategy:
-      fail-fast: true
-      matrix:
-        bitcoind-version: ["27.1"]
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v2
     - name: Test in FreeBSD
       id: test
-      uses: vmactions/freebsd-vm@v1
+      uses: vmactions/freebsd-vm@v0.1.5
       with:
         usesh: true
-        sync: rsync
-        copyback: false
         prepare: |
           pkg install -y \
-            bash \
             wget \
-            python310 \
+            py38-pip \
+            py38-sqlite3 \
             gmake \
             git \
             python \
-            postgresql16-server \
+            postgresql12-server \
             autoconf \
             automake \
             libtool \
@@ -41,29 +34,41 @@ jobs:
             gettext \
             sqlite3 \
             lowdown \
-            pkgconf \
-            jq \
-            protobuf \
             curl
 
-            python3.10 -m ensurepip
-            python3.10 -m pip install --upgrade pip
-            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
-
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2021-08-3z1
 
+          cd /tmp/ || exit 1
+          wget https://bitcoincore.org/bin/bitcoin-core-25.0/bitcoin-25.0-x86_64-linux-gnu.tar.gz
+          tar -xf bitcoin-25.0-x86_64-linux-gnu.tar.bz2
+          sudo mv bitcoin-25.0/bin/* /usr/local/bin
+          rm -rf \
+            bitcoin-25.0-x86_64-linux-gnu.tar.gz \
+            bitcoin-25.0
+          
         run: |
-          PATH=/root/.local/bin:$PATH:/root/.cargo/bin; export PATH
+          PATH=/root/.local/bin:$PATH
           pip install --user -U wheel pip
-          pip3 install --user poetry
-          poetry install
+          pip install --user -U -r requirements.txt
+
+          # Install utilities that aren't dependencies, but make
+          # running tests easier/feasible on CI (and pytest which
+          # keeps breaking the rerunfailures plugin).
+          pip install --user -U \
+            blinker \
+            flake8 \
+            mako \
+            pytest-sentry \
+            pytest-test-groups==1.0.3 \
+            pytest-custom-exit-code==0.3.0 \
+            pytest-json-report
 
           git clone https://github.com/lightning/bolts.git ../bolts
           # fatal: unsafe repository ('/Users/runner/work/lightning/lightning' is owned by someone else)
           git config --global --add safe.directory `pwd`
-          for d in libsodium libwally-core gheap jsmn libbacktrace lowdown; do git config --global --add safe.directory `pwd`/external/$d; done
           git submodule update --init --recursive
 
-          ./configure CC="$CC" --disable-valgrind
+          ./configure CC="$CC" --enable-developer --disable-valgrind
 
           cat config.vars
 
@@ -75,5 +80,8 @@ jobs:
           EOF
 
           # Just run a "quick" test without memory checking
-          poetry run gmake
+          gmake
+
+          # Clean up to maximize rsync's chances of succeeding
+          gmake clean
 

.github/workflows/ci.yaml

@@ -15,12 +15,11 @@ env:
   # of a bit of compile time.
   RUST_PROFILE: release
   SLOW_MACHINE: 1
-  CI_SERVER_URL: "http://35.239.136.52:3170"
 
 jobs:
   prebuild:
     name: Pre-build checks
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
     timeout-minutes: 30
     env:
       BOLTDIR: bolts
@@ -28,72 +27,42 @@ jobs:
       fail-fast: true
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
 
       - name: Rebase
-        # We can't rebase if we're on master already.
-        if: github.ref != 'refs/heads/master'
         run: |
           git config user.name github-actions
           git config user.email github-actions@github.com
           git fetch origin ${{ github.base_ref }}
           git rebase origin/${{ github.base_ref }}
 
-      - name: Check changelog
-        env:
-          PR_DESCRIPTION: "${{ github.event.pull_request.body || '' }}"
-          EVENT_NAME: "${{ github.event_name }}"
-          BASE_REF: "${{ github.base_ref || 'master' }}"
-        run: |
-          echo "Event Name: $EVENT_NAME"
-          echo "Base Ref: $BASE_REF"
-          echo "PR DESCRIPTION: $PR_DESCRIPTION"
-          if [ "$EVENT_NAME" = "pull_request" ]; then
-            if [[ "$PR_DESCRIPTION" != *"Changelog-None"* && \
-                  -z "$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')" && \
-                  "$(git rev-parse --abbrev-ref HEAD)" != "$BASE_REF" ]]; then
-              echo "::error::'Changelog' entry is missing in all commits, and 'Changelog-None' not specified in the PR description"
-              exit 1
-            else
-              if [[ "$PR_DESCRIPTION" == *"Changelog-None"* ]]; then
-                echo "Changelog found in PR description"
-              else
-                echo "Changelog found in Commit \"$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')\""
-              fi
-            fi
-          else
-            echo "Not a PR event."
-          fi
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: 3.8
 
       - name: Install dependencies
         run: |
           bash -x .github/scripts/setup.sh
           pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
           # Export and then use pip to install into the current env
           poetry export -o /tmp/requirements.txt --without-hashes --with dev
           pip install -r /tmp/requirements.txt
+          # Make sure we have clnrest requirements, for check-manpages
+          pip install -r plugins/clnrest/requirements.txt
           # We're going to check BOLT quotes, so get the latest version
           git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
       - name: Configure
         run: ./configure --enable-debugbuild --enable-rust
       - name: Check source
-        env:
-          VALGRIND: 0
-          PYTEST_OPTS: --timeout=1200
-        run: make check-source BASE_REF="origin/${{ github.base_ref }}"
+        run: make -j 4 check-source BASE_REF="origin/${{ github.base_ref }}"
       - name: Check Generated Files have been updated
-        run: make check-gen-updated
+        run: make -j 4 check-gen-updated
       - name: Check docs
-        run: make check-doc
+        run: make -j 4 check-doc
 
   compile:
     name: Compile CLN ${{ matrix.cfg }}
@@ -105,30 +74,34 @@ jobs:
       fail-fast: true
       matrix:
         include:
-          - CFG: compile-gcc
+          - CFG: gcc-dev1
+            DEVELOPER: 1
             VALGRIND: 1
             COMPILER: gcc
-          - CFG: compile-gcc-O3
+          - CFG: gcc-dev0
+            DEVELOPER: 0
             VALGRIND: 1
             COMPILER: gcc
             COPTFLAGS_VAR: COPTFLAGS="-O3 -Werror"
           # While we're at it let's try to compile with clang
-          - CFG: compile-clang
+          - CFG: clang-dev1
+            DEVELOPER: 1
             VALGRIND: 1
             COMPILER: clang
-          - CFG: compile-clang-sanitizers
+          - CFG: clang-sanitizers
+            DEVELOPER: 1
             COMPILER: clang
             ASAN: 1
             UBSAN: 1
             VALGRIND: 0
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: 3.8
 
       - name: Install dependencies
         run: |
@@ -136,6 +109,7 @@ jobs:
 
       - name: Build
         env:
+          DEVELOPER: ${{ matrix.DEVELOPER }}
           COMPILER: ${{ matrix.COMPILER }}
           ASAN: ${{ matrix.ASAN }}
           UBSAN: ${{ matrix.UBSAN }}
@@ -145,7 +119,6 @@ jobs:
         run: |
           set -e
           pip3 install --user pip wheel poetry
-          poetry self add poetry-plugin-export
           poetry export -o requirements.txt --with dev --without-hashes
           python3 -m pip install -r requirements.txt
           ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }}
@@ -156,7 +129,7 @@ jobs:
           mv testpack.tar.bz2 cln-${CFG}.tar.bz2
       - name: Check rust packages
         run: cargo test --all
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3
         with:
           name: cln-${{ matrix.CFG }}.tar.bz2
           path: cln-${{ matrix.CFG }}.tar.bz2
@@ -175,26 +148,24 @@ jobs:
       fail-fast: true
       matrix:
         include:
-          - CFG: compile-gcc
+          - CFG: gcc-dev1
             VALGRIND: 1
-          - CFG: compile-clang-sanitizers
+          - CFG: clang-sanitizers
             VALGRIND: 0
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: 3.8
 
       - name: Install dependencies
         run: |
           bash -x .github/scripts/setup.sh
-          sudo apt-get update -qq
           sudo apt-get install -y -qq lowdown
           pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
           # Export and then use pip to install into the current env
           poetry export -o /tmp/requirements.txt --without-hashes --with dev
           pip install -r /tmp/requirements.txt
@@ -202,14 +173,14 @@ jobs:
           git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
 
       - name: Download build
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
           name: cln-${{ matrix.CFG }}.tar.bz2
 
       - name: Check
         run: |
           tar -xaf cln-${{ matrix.CFG }}.tar.bz2
-          eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }}
+          make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }}
 
   check-fuzz:
     name: Run fuzz regression tests
@@ -218,25 +189,24 @@ jobs:
       - prebuild
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: 3.8
 
       - name: Install dependencies
         run: |
           bash -x .github/scripts/setup.sh
           pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
           # Export and then use pip to install into the current env
           poetry export -o /tmp/requirements.txt --without-hashes --with dev
           pip install -r /tmp/requirements.txt
 
       - name: Build
         run: |
-          ./configure --enable-debugbuild --enable-fuzzing --enable-address-sanitizer --enable-ub-sanitizer --disable-valgrind CC=clang
+          ./configure --enable-debugbuild --enable-fuzzing --enable-developer --disable-valgrind CC=clang
           make -j $(nproc) check-fuzz
 
   integration:
@@ -244,60 +214,74 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 120
     env:
+      BITCOIN_VERSION: "25.0"
+      ELEMENTS_VERSION: 22.0.2
       RUST_PROFILE: release  # Has to match the one in the compile step
       PYTEST_OPTS: --timeout=1200
     needs:
       - compile
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         include:
-          - NAME: gcc
-            CFG: compile-gcc
+          - NAME: gcc-dev1
+            CFG: gcc-dev1
+            DEVELOPER: 1
+            TEST_DB_PROVIDER: sqlite3
+            COMPILER: gcc
+            TEST_NETWORK: regtest
+          - NAME: gcc-dev0
+            CFG: gcc-dev0
+            DEVELOPER: 0
             TEST_DB_PROVIDER: sqlite3
             COMPILER: gcc
             TEST_NETWORK: regtest
           # While we're at it let's try to compile with clang
-          - NAME: clang
-            CFG: compile-clang
+          - NAME: clang-dev1
+            CFG: clang-dev1
+            DEVELOPER: 1
             TEST_DB_PROVIDER: sqlite3
             COMPILER: clang
             TEST_NETWORK: regtest
           # And of course we want to test postgres too
           - NAME: postgres
-            CFG: compile-gcc
+            CFG: gcc-dev1
+            DEVELOPER: 1
             COMPILER: gcc
             TEST_DB_PROVIDER: postgres
             TEST_NETWORK: regtest
           # And don't forget about elements (like cdecker did when
           # reworking the CI...)
           - NAME: liquid
-            CFG: compile-gcc
+            CFG: gcc-dev1
+            DEVELOPER: 1
             COMPILER: gcc
             TEST_NETWORK: liquid-regtest
             TEST_DB_PROVIDER: sqlite3
           # And dual funding!
           - NAME: dual-fund
-            CFG: compile-gcc
+            CFG: gcc-dev1
             TEST_DB_PROVIDER: sqlite3
             COMPILER: gcc
             TEST_NETWORK: regtest
+            DEVELOPER: 1
             EXPERIMENTAL_DUAL_FUND: 1
           # And splicing!
           - NAME: splicing
-            CFG: compile-gcc
+            CFG: gcc-dev1
             TEST_DB_PROVIDER: sqlite3
             COMPILER: gcc
             TEST_NETWORK: regtest
+            DEVELOPER: 1
             EXPERIMENTAL_SPLICING: 1
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: 3.8
 
       - name: Install dependencies
         run: |
@@ -305,30 +289,16 @@ jobs:
           poetry install
 
       - name: Install bitcoind
-        env:
-          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
         run: .github/scripts/install-bitcoind.sh
 
       - name: Download build
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
           name: cln-${{ matrix.CFG }}.tar.bz2
 
-      - name: Unpack pre-built CLN
-        env:
-          CFG: ${{ matrix.CFG }}
-        run: |
-          tar -xaf cln-${CFG}.tar.bz2
-
-      - name: Switch network
-        if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }}
-        run: |
-          # Loading the network from config.vars rather than the envvar is a terrible idea...
-          sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars
-          cat config.vars
-
       - name: Test
         env:
+          DEVELOPER: ${{ matrix.DEVELOPER }}
           COMPILER: ${{ matrix.COMPILER }}
           EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }}
           EXPERIMENTAL_SPLICING: ${{ matrix.EXPERIMENTAL_SPLICING }}
@@ -341,22 +311,23 @@ jobs:
           TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
           LIGHTNINGD_POSTGRES_NO_VACUUM: 1
         run: |
-          env
-          cat config.vars
-          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
+          tar -xaf cln-${CFG}.tar.bz2
+          VALGRIND=0 poetry run pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
 
   integration-valgrind:
     name: Valgrind Test CLN ${{ matrix.name }}
     runs-on: ubuntu-22.04
     timeout-minutes: 120
     env:
+      BITCOIN_VERSION: "25.0"
+      ELEMENTS_VERSION: 22.0.2
       RUST_PROFILE: release  # Has to match the one in the compile step
-      CFG: compile-gcc
+      CFG: gcc-dev1
       PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800
     needs:
       - compile
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         include:
           - NAME: Valgrind (01/10)
@@ -381,16 +352,15 @@ jobs:
             PYTEST_OPTS: --test-group=10 --test-group-count=10
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: 3.8
 
       - name: Install dependencies
         run: |
-          sudo apt-get update -qq
           sudo apt-get install -yyq valgrind
           pip3 install --user pip wheel poetry
           poetry install
@@ -399,25 +369,27 @@ jobs:
         run: .github/scripts/install-bitcoind.sh
 
       - name: Download build
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
-          name: cln-compile-gcc.tar.bz2
+          name: cln-gcc-dev1.tar.bz2
 
       - name: Unpack build
-        run: tar -xvjf cln-compile-gcc.tar.bz2
+        run: tar -xvjf cln-gcc-dev1.tar.bz2
 
       - name: Test
         env:
           SLOW_MACHINE: 1
           TEST_DEBUG: 1
         run: |
-          VALGRIND=1 poetry run eatmydata pytest tests/ -vvv -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
+          VALGRIND=1 poetry run pytest tests/ -vvv -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
 
   integration-sanitizers:
     name: Sanitizers Test CLN
     runs-on: ubuntu-22.04
     timeout-minutes: 120
     env:
+      BITCOIN_VERSION: "25.0"
+      ELEMENTS_VERSION: 22.0.2
       RUST_PROFILE: release
       SLOW_MACHINE: 1
       TEST_DEBUG: 1
@@ -425,7 +397,7 @@ jobs:
     needs:
       - compile
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         include:
           - NAME: ASan/UBSan (01/10)
@@ -450,204 +422,49 @@ jobs:
             PYTEST_OPTS: --test-group=10 --test-group-count=10
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: 3.8
 
       - name: Install dependencies
         run: |
           bash -x .github/scripts/setup.sh
           set -e
-          pip3 install --user wheel poetry
-          poetry install --with dev --no-root
-
-      - name: Install bitcoind
-        run: .github/scripts/install-bitcoind.sh
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-compile-clang-sanitizers.tar.bz2
-
-      - name: Unpack build
-        run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2
-
-      - name: Test
-        run: |
-          poetry run eatmydata pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
-
-  update-docs-examples:
-    name: Update examples in doc schemas (disabled temporarily!)
-    if: false
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-    env:
-      VALGRIND: 0
-      GENERATE_EXAMPLES: 1
-      PYTEST_OPTS: --timeout=1200
-      TEST_NETWORK: regtest
-    needs:
-      - compile
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-      - name: Install bitcoind
-        env:
-          TEST_NETWORK: regtest
-        run: .github/scripts/install-bitcoind.sh
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-compile-gcc.tar.bz2
-      - name: Unpack pre-built CLN
-        run: |
-          tar -xaf cln-compile-gcc.tar.bz2
-      - name: Test
-        run: |
-          eatmydata make -j $(nproc) check-doc-examples
-
-  min-btc-support:
-    name: Test minimum supported BTC v${{ matrix.MIN_BTC_VERSION }} with ${{ matrix.NAME }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    env:
-      RUST_PROFILE: release  # Has to match the one in the compile step
-      PYTEST_OPTS: --timeout=1200
-    needs:
-      - compile
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - NAME: clang
-            CFG: compile-clang
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: clang
-            TEST_NETWORK: regtest
-            MIN_BTC_VERSION: '25.0'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
           pip3 install --user pip wheel poetry
+          poetry export -o requirements.txt --with dev --without-hashes
+          python3 -m pip install -r requirements.txt
           poetry install
 
-      - name: Download Bitcoin Core
-        run: wget "https://bitcoincore.org/bin/bitcoin-core-${{ matrix.MIN_BTC_VERSION }}/bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz"
-
-      - name: Extract Bitcoin Core
-        run: tar -xf "bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz"
-
-      - name: Move Bitcoin Core Binaries
-        run: sudo mv bitcoin-${{ matrix.MIN_BTC_VERSION }}/bin/* /usr/local/bin/
-
-      - name: Clean Up Downloaded Bitcoin
-        run: rm -rf "bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz" "bitcoin-${{ matrix.MIN_BTC_VERSION }}"
+      - name: Install bitcoind
+        run: .github/scripts/install-bitcoind.sh
 
       - name: Download build
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
+          name: cln-clang-sanitizers.tar.bz2
 
-      - name: Unpack pre-built CLN
-        env:
-          CFG: ${{ matrix.CFG }}
-        run: |
-          tar -xaf cln-${CFG}.tar.bz2
+      - name: Unpack build
+        run: tar -xvjf cln-clang-sanitizers.tar.bz2
 
       - name: Test
-        env:
-          COMPILER: ${{ matrix.COMPILER }}
-          COMPAT: 1
-          CFG: ${{ matrix.CFG }}
-          SLOW_MACHINE: 1
-          PYTEST_PAR: 10
-          TEST_DEBUG: 1
-          TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
-          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
-          LIGHTNINGD_POSTGRES_NO_VACUUM: 1
         run: |
-          env
-          cat config.vars
-          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
-
-  check-flake:
-    name: Check Nix Flake
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: true
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: 'recursive'
-      - name: Check Nix flake inputs
-        uses: DeterminateSystems/flake-checker-action@v8
-      - name: Install Nix
-        uses: cachix/install-nix-action@V27
-        with:
-          nix_path: nixpkgs=channel:nixos-24.05
-      - name: Check flake
-        run: nix flake check .?submodules=1#
+          poetry run pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
 
   gather:
     # A dummy task that depends on the full matrix of tests, and
     # signals successful completion. Used for the PR status to pass
-    # before merging.  Needs to run even if they failed!
+    # before merging.
     name: CI completion
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
     needs:
       - integration
       - check-units
       - integration-valgrind
       - integration-sanitizers
-      - min-btc-support
-      - check-flake
-    if: ${{ always() }}
     steps:
       - name: Complete
-        env:
-          JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC FLAKE"
-          INTEGRATION: ${{ needs.integration.result }}
-          CHECK_UNITS: ${{ needs['check-units'].result }}
-          VALGRIND: ${{ needs['integration-valgrind'].result }}
-          SANITIZERS: ${{ needs['integration-sanitizers'].result }}
-          DOCS: ${{ needs['update-docs-examples'].result }}
-          BTC: ${{ needs['min-btc-support'].result }}
-          FLAKE: ${{ needs['check-flake'].result }}
         run: |
-          failed=""
-          for name in $JOB_NAMES; do
-            result="${!name}"
-            echo "$name: $result"
-            if [[ "$result" != "success" ]]; then
-              failed="yes"
-            fi
-          done
-          if [[ "$failed" == "yes" ]]; then
-            echo "One or more required jobs failed"
-            exit 1
-          fi
+          echo CI completed successfully

.github/workflows/ci_build.yml

@@ -0,0 +1,18 @@
+name: CI Compilation testing
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { OS: alpine }
+    steps:
+      - uses: actions/checkout@v2
+      - name: Integration testing
+        run: |
+          docker build -f contrib/docker/Dockerfile.${{matrix.OS}} -t clightning-${{matrix.OS}} .

.github/workflows/crate-io.yml

@@ -1,30 +0,0 @@
----
-name: "Release Rust 🦀"
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - "master"
-
-jobs:
-  release_rust:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@stable
-      - uses: arduino/setup-protoc@v3
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './cln-rpc'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './plugins'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './cln-grpc'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true

.github/workflows/docker-release.yml

@@ -1,118 +0,0 @@
-name: Build and push multi-platform docker images
-
-on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      version:
-        description: 'Release version'
-        required: true
-
-      repository-name:
-        description: 'Docker repository name'
-        default: 'elementsproject'
-        required: false
-
-      platforms-to-build:
-        description: 'List of platforms to build'
-        default: 'linux/amd64,linux/arm64,linux/arm/v7'
-        required: false
-  
-      push-latest:
-        description: 'Push the latest tag also?'
-        default: 'false'
-        required: false
-
-jobs:
-  build:
-    runs-on: ubuntu-22.04
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-      with:
-        ref: ${{ github.ref }}  # Ensures the branch triggering the workflow is checked out
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-
-    - name: Setup Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
-    - name: Log in to Docker Hub
-      uses: docker/login-action@v3
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
-    - name: Set up values
-      id: set-values
-      run: |
-        if [[ "${{ github.event.inputs.version }}" != "" ]]; then
-          echo "Input version provided"
-          VERSION=${{ github.event.inputs.version }}
-        elif [[ ${{ github.ref_type }} == "tag" ]]; then
-          echo "This is a tag event"
-          VERSION=${{ github.ref_name }}
-        else
-          echo "No version provided and no tag found."
-          exit 1
-        fi
-        echo "VERSION=$VERSION" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.repository-name }}" != "" ]]; then
-          REPONAME=${{ github.event.inputs.repository-name }}
-        else
-          REPONAME="elementsproject"
-        fi
-        echo "REPONAME=$REPONAME" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.platforms-to-build }}" != "" ]]; then
-          PLATFORMS=${{ github.event.inputs.platforms-to-build }}
-        else
-          PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
-        fi
-        echo "PLATFORMS=$PLATFORMS" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.push-latest }}" == "true" ]] || 
-          ([[ "${{ github.ref_type }}" == "tag" ]] && [[ ! "$VERSION" =~ rc ]]); then
-          echo "Latest true"
-          PUSHLATEST="true"
-        else
-          echo "Latest false"
-          PUSHLATEST="false"
-        fi
-        echo "PUSHLATEST=$PUSHLATEST" >> $GITHUB_ENV
-
-        TAGS="$REPONAME/lightningd:$VERSION"
-        if [[ "$PUSHLATEST" == "true" ]]; then
-          TAGS="$TAGS,$REPONAME/lightningd:latest"
-        fi
-        echo "TAGS=$TAGS" >> $GITHUB_ENV
-
-    - name: Print GitHub Ref Values
-      run: |
-        echo "GITHUB REF TYPE: ${{ github.ref_type }}"
-        echo "GITHUB REF NAME: ${{ github.ref_name }}"
-        echo "EVENT INPUT VERSION: ${{ github.event.inputs.version }}"
-        echo "EVENT INPUT REPO: ${{ github.event.inputs.repository-name }}"
-        echo "EVENT INPUT PLATFORMS: ${{ github.event.inputs.platforms-to-build }}"
-        echo "EVENT INPUT PUSH LATEST: ${{ github.event.inputs.push-latest }}"
-        echo "ENV VERSION: ${{ env.VERSION }}"
-        echo "ENV REPO NAME: ${{ env.REPONAME }}"
-        echo "ENV PLATFORMS: ${{ env.PLATFORMS }}"
-        echo "ENV PUSH LATEST: ${{ env.PUSHLATEST }}"
-        echo "ENV TAGS: ${{ env.TAGS }}"
-
-    - name: Build and push Docker image
-      uses: docker/build-push-action@v5
-      with:
-        context: .
-        file: ./Dockerfile
-        push: true
-        platforms: ${{ env.PLATFORMS }}
-        tags: ${{ env.TAGS }}

.github/workflows/docker.yml

@@ -1,34 +0,0 @@
-name: Docker packaging
-on:
-  push:
-    tags:
-      - 'basedon-*'
-
-jobs:
-  main:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Environment variables
-        run: env
-      -
-        name: Create images
-        env:
-          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
-          DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }}
-          DOCKERHUB_REPO: ${{ vars.DOCKERHUB_REPO }}
-        shell: bash
-        run: |
-          LATEST_TAG=${GITHUB_REF#refs/tags/}
-          LATEST_TAG=${LATEST_TAG:8} #trim "basedon-" from tag
-          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USER" --password-stdin
-          docker buildx create --use
-          DOCKER_BUILDX_OPTS="--platform linux/amd64,linux/arm64,linux/arm/v7 --push"
-          docker buildx build $DOCKER_BUILDX_OPTS -t $DOCKERHUB_REPO:$LATEST_TAG .

.github/workflows/macos.yaml

@@ -5,58 +5,55 @@ on:
 jobs:
   smoke-test:
     name: Smoke Test macOS
-    runs-on: macos-14
+    runs-on: macos-latest
     timeout-minutes: 120
     strategy:
       fail-fast: true
-      matrix:
-        bitcoind-version: ["27.1"]
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Download Bitcoin ${{ matrix.bitcoind-version }} & install binaries
-        run: |
-          export BITCOIND_VERSION=${{ matrix.bitcoind-version }}
-          export TARGET_ARCH="arm64-apple-darwin"
-
-          wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
-          tar -xzf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
-          sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin
-          rm -rf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz bitcoin-${BITCOIND_VERSION}
+        uses: actions/checkout@v2.0.0
 
       - name: Install dependencies
         run: |
-          export PATH="/usr/local/opt:/Users/runner/.local/bin:/opt/homebrew/bin/python3.10/bin:$PATH"
+          export PATH="/usr/local/opt:/Users/runner/.local/bin:/Users/runner/Library/Python/3.10/bin:$PATH"
 
-          brew install gnu-sed python@3.10 autoconf automake libtool protobuf
-          python3.10 -m pip install -U --user poetry==1.8.0 wheel pip mako
+          export BITCOIN_VERSION=25.0
+          brew install wget autoconf automake libtool python@3.10 gnu-sed gettext libsodium
+
+          (
+            cd /tmp/
+            wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-osx64.tar.gz -O bitcoin.tar.gz
+            tar -xvzf bitcoin.tar.gz
+            sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin
+          )
+
+          python3.10 -m pip install -U --user poetry wheel pip
           python3.10 -m poetry install
+          python3.10 -m pip install -U --user mako
+          
+          ln -s /usr/local/Cellar/gettext/0.20.1/bin/xgettext /usr/local/opt
 
-      - name: Build and install CLN
+      - name: Build
+        env:
+          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
+          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
+          NO_PYTHON: ${{ matrix.NO_PYTHON }}
+          COPTFLAGS: ${{ matrix.COPTFLAGS }}
+          NETWORK: ${{ matrix.NETWORK }}
+          TEST_CMD: ${{ matrix.TEST_CMD }}
+          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
+          TEST_GROUP: ${{ matrix.TEST_GROUP }}
         run: |
-          export CPATH=/opt/homebrew/include
-          export LIBRARY_PATH=/opt/homebrew/lib
+          export PATH="/usr/local/opt:/Users/runner/.local/bin:/Users/runner/Library/Python/3.10/bin:/usr/local/opt:$PATH"
+          export LDFLAGS="-L/usr/local/opt/sqlite/lib"
+          export CPPFLAGS="-I/usr/local/opt/sqlite/include"
 
-          python3.10 -m poetry run ./configure --disable-valgrind --disable-compat
+          cat << EOF > pytest.ini
+          [pytest]
+          addopts=-p no:logging --color=yes --timeout=600 --timeout-method=thread --test-group-random-seed=42 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+          markers =
+              slow_test: marks tests as slow (deselect with '-m "not slow_test"')
+          EOF
+
+          python3.10 -m poetry run ./configure --enable-developer --disable-valgrind --disable-compat
           python3.10 -m poetry run make
-
-      - name: Start bitcoind in regtest mode
-        run: |
-          bitcoind -regtest -daemon
-          sleep 5
-
-      - name: Generate initial block
-        run: |
-          bitcoin-cli -regtest createwallet default_wallet
-          bitcoin-cli -regtest generatetoaddress 1 $(bitcoin-cli -regtest getnewaddress)
-          sleep 2
-
-      - name: Start CLN in regtest mode
-        run: |
-          lightningd/lightningd --network=regtest --log-file=/tmp/l1.log --daemon
-          sleep 5
-
-      - name: Verify CLN is running
-        run: |
-          cli/lightning-cli --regtest getinfo

.github/workflows/prototest.yaml

@@ -0,0 +1,44 @@
+---
+name: LN Proto Test
+on:
+  push:
+    branches:
+      - "master"
+  pull_request:
+jobs:
+  proto-test:
+    name: Protocol Test Config
+    runs-on: ubuntu-22.04
+    timeout-minutes: 120
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - {compiler: clang, db: sqlite3}
+          - {compiler: gcc, db: postgres}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build and run
+        run: |
+          docker build -f contrib/docker/Dockerfile.ubuntu -t cln-ci-ubuntu .
+          docker run -e ARCH=${{ matrix.arch }} \
+          -e COMPILER=${{ matrix.compiler }} \
+          -e DB=${{ matrix.db }} \
+          -e NETWORK=${{ matrix.network }} \
+          -e TARGET_HOST=${{ matrix.TARGET_HOST }} \
+          -e VALGRIND=${{ matrix.valgrind }} \
+          -e DEVELOPER=1 \
+          -e PYTEST_PAR=2 \
+          -e PYTEST_OPTS="--timeout=300" \
+          -e TEST_CMD="make check-protos" \
+          -e TEST_GROUP=1 \
+          -e TEST_GROUP_COUNT=1 \
+          -e TIMEOUT=120 \
+          cln-ci-ubuntu
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2.2.4
+        with:
+          name: Junit Report ${{ github.run_number }}.{{ matrix.cfg }}
+          path: report.*

.github/workflows/pypi.yml

@@ -1,22 +1,19 @@
+---
 name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
-
 on:
+  # Only deploy if we're the result of a PR being merged
   push:
+    branches:
+      - master
     tags:
-      - 'v[0-9]+.[0-9]+'
+      # Semantic versioning tags
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      dist-location:
-        description: 'Distribution location (test/prod)'
-        default: 'test'
-        required: false
-
+      # Date style tags
+      - 'v[0-9]{2}.[0-9]{2}'
 jobs:
   deploy:
     name: Build and publish ${{ matrix.package }} 🐍
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
     timeout-minutes: 120
     strategy:
       fail-fast: true
@@ -28,74 +25,62 @@ jobs:
             WORKDIR: contrib/pyln-testing
           - PACKAGE: pyln-proto
             WORKDIR: contrib/pyln-proto
+          - PACKAGE: pyln-grpc-proto
+            WORKDIR: contrib/pyln-grpc-proto
+          # Bolt packages are handled differently
+          #- PACKAGE: pyn-bolt1
+          #  WORKDIR: contrib/pyln-spec/bolt1/
+          #- PACKAGE: pyn-bolt2
+          #  WORKDIR: contrib/pyln-spec/bolt2/
+          #- PACKAGE: pyn-bolt4
+          #  WORKDIR: contrib/pyln-spec/bolt4/
+          #- PACKAGE: pyn-bolt7
+          #  WORKDIR: contrib/pyln-spec/bolt7/
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+    - uses: actions/checkout@master
       with:
         # Need to fetch entire history in order to locate the version tag
         fetch-depth: 0
 
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+
+    - name: Install pypa/build and poetry
+      run: >-
+        python -m pip install build poetry --user
+
     - name: Check version tag
       run: >-
-        git describe --tags --always --dirty=-modded --abbrev=7
+        git describe --always --dirty=-modded --abbrev=7
 
-    - name: Setup Version
+    - name: Build a binary wheel and a source tarball
       env:
         WORKDIR: ${{ matrix.WORKDIR }}
       run: |
-        echo "VERSION=$(git describe --tags --abbrev=0).post$(git describe --tags --abbrev=1 | awk -F "-" '{print $2}')" >> $GITHUB_ENV
-
-    - name: Set up values
-      id: set-values
-      run: |
-        if [[ "${{ github.event.inputs.dist-location }}" != "" ]]; then
-          DISTLOCATION=${{ github.event.inputs.dist-location }}
-        elif [[ "${{ github.ref_type }}" == "tag" ]] && [[ ! "${{ github.ref_name }}" =~ rc ]]; then
-          DISTLOCATION="prod"
-        else
-          DISTLOCATION="test"
-        fi
-        echo "DISTLOCATION=$DISTLOCATION" >> $GITHUB_OUTPUT
-        echo "EVENT DISTLOCATION: ${{ github.event.inputs.dist-location }}"
-        echo "DISTRIBUTION LOCATION: $DISTLOCATION"
-  
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.10'
-
-    - name: Install Poetry
-      run: |
-        curl -sSL https://install.python-poetry.org | python3 -
-        echo "$HOME/.local/bin" >> $GITHUB_PATH
-        echo "PATH=$HOME/.local/bin:$PATH"
+        export VERSION=$(git describe --abbrev=0).post$(git describe --abbrev=1 | awk -F "-" '{print $2}')
+        cd ${{ env.WORKDIR}}
+        make upgrade-version NEW_VERSION=$VERSION
+        poetry build
 
     - name: Publish distribution 📦 to Test PyPI
-      if: github.repository == 'ElementsProject/lightning' && steps.set-values.outputs.DISTLOCATION == 'test'
+      if: github.repository == 'ElementsProject/lightning'
       env:
         POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.TEST_PYPI_API_TOKEN }}
         WORKDIR: ${{ matrix.WORKDIR }}
       run: |
-        echo "POETRY VERSION TEST: $(poetry --version)"
-        echo "Pyln VERSION: $VERSION"
-        cd ${{ env.WORKDIR }}
-        python3 -m pip config set global.timeout 150
+        cd ${{ env.WORKDIR}}
         poetry config repositories.testpypi https://test.pypi.org/legacy/
-        make upgrade-version NEW_VERSION=$VERSION
-        poetry build  --no-interaction
-        poetry publish --repository testpypi --no-interaction --skip-existing
+        poetry publish --repository testpypi --no-interaction
 
     - name: Publish distribution 📦 to PyPI
-      if: github.repository == 'ElementsProject/lightning' && steps.set-values.outputs.DISTLOCATION == 'prod'
+      if: startsWith(github.ref, 'refs/tags') && github.repository == 'ElementsProject/lightning'
       env:
         POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
         WORKDIR: ${{ matrix.WORKDIR }}
       run: |
-        echo "POETRY VERSION PUBLISH: $(poetry --version)"
-        cd ${{ env.WORKDIR }}
-        export VERSION=$(git describe --tags --abbrev=0)
-        echo "Pyln VERSION: $VERSION"
+        cd ${{ env.WORKDIR}}
+        export VERSION=$(git describe --abbrev=0)
         make upgrade-version NEW_VERSION=$VERSION
-        python3 -m pip config set global.timeout 150
-        poetry build  --no-interaction
         poetry publish --no-interaction

.github/workflows/rdme-docs-sync.yml

@@ -6,14 +6,13 @@ on:
       - 'master'
     paths:
       - 'doc/**'
-  workflow_dispatch:
 
 jobs:
   rdme-docs-sync:  
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Check out repo 📚
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       - name: Sync doc/getting-started/ 🚀
         uses: readmeio/rdme@v8

.github/workflows/readme-rpc-sync.yml

@@ -5,39 +5,24 @@ on:
     branches:
       - 'master'
     paths:
-      - 'doc/schemas/*.json'
-      - 'doc/*.1.md'
-      - 'doc/*.5.md'
-      - 'doc/*.8.md'
-      - 'doc/lightningd-*.7.md'
-      - 'doc/reckless.7.md'
-  workflow_dispatch:
+      - 'doc/**.md'
 
 jobs:
   rdme-rpc-sync:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
 
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v2
       with:
-        python-version: 3.9
+        python-version: '3.8'
 
-    - name: Install python modules
-      run: |
-        python -m pip install requests mako grpcio-tools
-
-    - name: Install dependencies
-      run: bash -x .github/scripts/setup.sh
-
-    - name: Build (including rpc .md files)
-      run: |
-        ./configure --enable-debugbuild
-        make -j $(nproc)
+    - name: Install requests module
+      run: python -m pip install requests
 
     - name: Set environment variable and run
       env:
-        README_API_KEY: ${{ secrets.README_API_KEY }}
+        README_API_KEY: ${{ secrets.README_API_KEY }}  
       run: python .github/scripts/sync-rpc-cmds.py

.github/workflows/release.yml

@@ -1,191 +0,0 @@
----
-# https://docs.corelightning.org/docs/release-checklist
-name: "Release 🚀"
-on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      version:
-        description: 'Release version'
-        required: true
-      create_release:
-        description: Create a draft release
-        default: no
-        type: choice
-        options:
-          - yes
-          - no
-
-jobs:
-  check:
-    name: Check
-    outputs:
-      version: ${{ steps.capture.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.ref }}
-          fetch-depth: 0
-          fetch-tags: true
-
-      - name: Determine version
-        run: |
-          if [[ "${{ github.event.inputs.version }}" != "" ]]; then
-            VERSION="${{ github.event.inputs.version }}"
-          elif [ "${{ github.ref_type }}" == "tag" ]; then
-            VERSION="${{ github.ref_name }}"
-          else
-            echo "No release version provided and no tag found."
-            exit 1
-          fi
-          echo "VERSION=$VERSION" >> "$GITHUB_ENV"
-          echo "Determined version: $VERSION"
-
-      - name: Validate release
-        run: tools/check-release.sh --version=${VERSION}
-
-      - name: Catpure version output
-        id: capture
-        run: echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
-
-  releases:
-    name: Releases
-    needs: check
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false    # Let each build finish.
-      matrix:
-        target:
-          - 'bin-Fedora'
-          - 'bin-Ubuntu-focal'
-          - 'bin-Ubuntu-jammy'
-          - 'bin-Ubuntu-noble'
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-tags: true
-
-      # tools/build-release.sh requires lowdown
-      - name: Prepare base environment
-        run: |
-          sudo apt-get install -y lowdown
-          ./configure
-
-      - name: Build environment setup
-        run: |
-          distribution=$(echo ${{ matrix.target }} | cut -d'-' -f3)
-          echo "Building base image for ${distribution}"
-          sudo docker run --rm -v $(pwd):/build ubuntu:${distribution} bash -c "\
-              apt-get update && \
-              apt-get install -y debootstrap && \
-              debootstrap ${distribution} /build/${distribution}"
-          sudo tar -C ${distribution} -c . | docker import - ${distribution}
-
-          # Build Docker image
-          docker build -t cl-repro-${distribution} - < contrib/reprobuild/Dockerfile.${distribution}
-        if: contains(matrix.target, 'Ubuntu')
-
-      - name: Build release
-        run: tools/build-release.sh ${{ matrix.target }}
-
-      - name: Upload target artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          path: release/
-          name: ${{ matrix.target }}
-          if-no-files-found: error
-
-  artifact:
-    name: Construct release artifact
-    needs:
-      - check
-      - releases
-    env:
-      version: ${{ needs.check.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Merge artifacts
-        uses: actions/upload-artifact/merge@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          pattern: bin-*
-          delete-merged: true
-
-  release:
-    name: Sign and prepare release draft
-    needs:
-      - check
-      - artifact
-    env:
-      version: ${{ needs.check.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-tags: true
-
-      - name: Download artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          path: release/
-
-      - name: Import GPG keys
-        id: gpg
-        uses: crazy-max/ghaction-import-gpg@v6
-        with:
-          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.GPG_PASSPHRASE }}
-          trust_level: 5
-
-      - name: Set default GPG key
-        run: echo "default-key ${{ steps.gpg.outputs.keyid }}" >> ~/.gnupg/gpg.conf
-
-      - name: Sign release
-        run: |
-          sudo apt-get install -y lowdown
-          ./configure
-          tools/build-release.sh --without-zip sign
-          mv release/SHA256SUMS.asc${{ steps.gpg.outputs.keyid  }} release/SHA256SUMS.asc
-
-      - name: Upload signed artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          overwrite: true
-          path: release/
-
-      - name: Determine release data
-        id: release_data
-        run: |
-          VERSION=${{ env.version }}
-          CHANGELOG_VERSION=${VERSION#v}
-          echo "CHANGELOG_VERSION=$CHANGELOG_VERSION"
-          echo "changelog_version=$CHANGELOG_VERSION" >> "$GITHUB_OUTPUT"
-
-          CHANGELOG_TITLE=$(grep "## \[${CHANGELOG_VERSION}\]" CHANGELOG.md)
-          echo "CHANGELOG_TITLE=$CHANGELOG_TITLE"
-          echo "changelog_title=$CHANGELOG_TITLE" >> "$GITHUB_OUTPUT"
-
-          RELEASE_TITLE=$(echo $CHANGELOG_TITLE | cut -d'"' -f2)
-          echo "RELEASE_TITLE=$RELEASE_TITLE"
-          echo "release_title=$RELEASE_TITLE" >> "$GITHUB_OUTPUT"
-
-      - name: Prepare release draft
-        if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.create_release == 'yes')
-        uses: softprops/action-gh-release@v2
-        with:
-          name: "${{ env.version }} ${{ steps.release_data.outputs.release_title }}"
-          tag_name: ${{ env.version }}
-          draft: true
-          prerelease: contains(env.version, "-rc")
-          files: release/*
-          fail_on_unmatched_files: true

.github/workflows/repro.yml

@@ -1,113 +0,0 @@
----
-# https://docs.corelightning.org/docs/repro
-name: Repro Build Nightly
-on:
-  # 05:00 Berlin, 03:00 UTC, 23:00 New York, 20:00 Los Angeles
-  schedule:
-    - cron: "0 3 * * *"
-  workflow_dispatch:
-
-jobs:
-  ubuntu:
-    name: "Ubuntu repro build: ${{ matrix.version }}"
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: false    # Let each build finish.
-      matrix:
-        version: ['focal', 'jammy', 'noble']
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-
-      - name: Build environment setup - ${{ matrix.version }}
-        run: |
-          echo "Building base image for ${{ matrix.version }}"
-          echo "STEP=Build environment setup" >> "$GITHUB_ENV"
-          sudo docker run --rm -v $(pwd):/build ubuntu:${{ matrix.version }} bash -c "\
-            apt-get update && \
-            apt-get install -y debootstrap && \
-            debootstrap ${{ matrix.version }} /build/${{ matrix.version }}"
-          sudo tar -C ${{ matrix.version }} -c . | docker import - ${{ matrix.version }}
-
-      - name: Builder image setup - ${{ matrix.version }}
-        run: |
-          echo "STEP=Builder image setup" >> "$GITHUB_ENV"
-          docker build -t cl-repro-${{ matrix.version }} - < contrib/reprobuild/Dockerfile.${{ matrix.version }}
-
-      - name: Build reproducible image and store Git state - ${{ matrix.version }}
-        run: |
-          echo "STEP=Build reproducible image and store Git state" >> "$GITHUB_ENV"
-
-          # Create release directory.
-          mkdir $GITHUB_WORKSPACE/release
-
-          # Perform the repro build.
-          docker run --name cl-build -v $GITHUB_WORKSPACE:/repo -e FORCE_MTIME=$(date +%F) -t cl-repro-${{ matrix.version }}
-
-          # Commit the image in order to inspect the build later.
-          docker commit cl-build cl-repro
-
-          # Inspect the version.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "make version > /repo/release/version.txt"
-
-          # Inspect the Git tree state.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "\
-              git --no-pager status > /repo/release/git.log && \
-              git --no-pager diff >> /repo/release/git.log"
-
-          # Change permissions on the release files for access by the runner environment.
-          sudo chown -R runner $GITHUB_WORKSPACE/release
-
-      - name: Assert clean version - ${{ matrix.version }}
-        run: |
-          echo "STEP=Assert clean version" >> "$GITHUB_ENV"
-          echo 'Version:'
-          cat release/version.txt
-          echo -e
-
-          releasefile=$(ls release/clightning-*)
-          echo 'Release file:'
-          ls -al release/clightning-*
-          echo -e
-          if [ -n "$(cat release/version.txt | sed -n '/-modded/p')" ] || \
-             [ -n "$(echo $releasefile | sed -n '/-modded/p')" ]; then
-            echo "Git Status and Diff:"
-            cat release/git.log
-            echo -e
-            echo 'Error: release modded / dirty tree.'
-            exit 1
-          else
-            echo 'Success! Clean release.'
-          fi
-
-      - name: Upload release artifact - ${{ matrix.version }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: release-${{ matrix.version }}
-          path: release
-          retention-days: 3 # Automatically delete after 3 days
-
-      - name: Send email on failure
-        if: ${{ failure() }}
-        uses: dawidd6/action-send-mail@v3
-        with:
-          server_address: smtp.gmail.com
-          server_port: 587
-          username: ${{ secrets.EMAIL_USERNAME }}
-          password: ${{ secrets.EMAIL_PASSWORD }}
-          from: ${{ secrets.EMAIL_USERNAME }}
-          to: ${{ vars.DISTRIBUTION_LIST }}
-          subject: "CI Failure: Step ${{ env.STEP }} failed for distro ${{ matrix.version }}"
-          convert_markdown: true
-          html_body: |
-            <html>
-              <body>
-                <p>GitHub Workflow ${{ github.workflow }} Failed! For more details, click on the action below.</p>
-                <strong>Failure Details:</strong><br/>
-                <strong>Event: </strong>${{ github.event_name }}<br/>
-                <strong>Job: </strong>${{ github.job }}<br/>
-                <strong>Distro: </strong>${{ matrix.version }}<br/>
-                <strong>Step: </strong>${{ env.STEP }}<br/>
-                <strong>Action: </strong><a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}</a><br/>
-              </body>
-            </html>

.gitignore

@@ -57,9 +57,7 @@ tests/plugins/test_selfdisable_after_getmanifest
 
 # Ignore generated files
 devtools/features
-doc/schemas/sql.json
-doc/*.7.md
-doc/*.[1578]
+doc/lightning*.[1578]
 doc/reckless*.[1578]
 *_sqlgen.[ch]
 *_wiregen.[ch]
@@ -69,36 +67,22 @@ tests/node_pb2.py
 tests/node_pb2_grpc.py
 tests/primitives_pb2.py
 tests/primitives_pb2_grpc.py
-tests/autogenerate-examples-status.log
-tests/autogenerate-examples.json
-tests/autogenerate-examples-repeat.log
 
 # Ignore unrelated stuff
 .DS_Store
 .gdb_history
 .python-version
-compile_commands.json
 
 # Rust targets
 target
 plugins/cln-grpc
-plugins/clnrest
 
 # Build directories
 bionic/
 focal/
 jammy/
-noble/
 release/
 .vscode/
-.cache/
 
 # Ignore release verification Sha256Sums
 SHA256SUMS-*
-doc/.doc_version
-autogenerate-examples-status.log
-
-# Ignore nix outputs
-result
-result-[0-9]*
-tests/plugins/channeld_fakenet

.gitmodules

@@ -14,6 +14,9 @@
 [submodule "external/gheap"]
 	path = external/gheap
 	url = https://github.com/valyala/gheap
+[submodule "external/lnprototest"]
+	path = external/lnprototest
+	url = https://github.com/rustyrussell/lnprototest.git
 [submodule "external/lowdown"]
 	path = external/lowdown
 	url = https://github.com/kristapsdz/lowdown.git

.pre-commit-config.yaml

@@ -1,13 +0,0 @@
-repos:
-- repo: https://github.com/astral-sh/ruff-pre-commit
-  # Ruff version.
-  rev: v0.8.0
-  hooks:
-    # Run the linter.
-    - id: ruff
-      args: [ --diff ]
-      exclude: "contrib/pyln-grpc-proto/pyln/grpc/(primitives|node)_pb2(|_grpc).py"
-    # Run the formatter.
-    - id: ruff-format
-      args: [ --diff ]
-      exclude: "contrib/pyln-grpc-proto/pyln/grpc/(primitives|node)_pb2(|_grpc).py"

.version

@@ -1 +0,0 @@
-25.05

Cargo.toml

@@ -2,12 +2,9 @@
 strip = "debuginfo"
 
 [workspace]
-resolver = "2"
 members = [
-    "cln-rpc",
-    "cln-grpc",
-    "plugins",
-    "plugins/grpc-plugin",
-    "plugins/rest-plugin",
-    "plugins/lsps-plugin",
+  "cln-rpc",
+  "cln-grpc",
+  "plugins",
+  "plugins/grpc-plugin",
 ]

Dockerfile

@@ -1,35 +1,11 @@
-# This Dockerfile is used by buildx to build ARM64, AMD64, and ARM32 Docker images from an AMD64 host.
-# To speed up the build process, we are cross-compiling rather than relying on QEMU.
-# There are four main stages:
-# * downloader: Downloads specific binaries needed for core lightning for each architecture.
-# * builder: Cross-compiles for each architecture.
-# * builder-python: Builds Python dependencies for wss-proxy with QEMU.
-# * final: Creates the runtime image.
-
-ARG DEFAULT_TARGETPLATFORM="linux/amd64"
-ARG BASE_DISTRO="debian:bookworm-slim"
-
-FROM --platform=$BUILDPLATFORM ${BASE_DISTRO} AS base-downloader
-RUN set -ex \
-	&& apt-get update \
-	&& apt-get install -qq --no-install-recommends ca-certificates dirmngr wget qemu-user-static binfmt-support
-
-FROM base-downloader AS base-downloader-linux-amd64
-ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu
-ENV DESCHASHPLUGIN_ARCH=linux-amd64
-ENV DESCHASHPLUGIN_HASH=deadc00c68fac80b2718d92f69bf06acd8fff646228d497bbb76a4f0a12ca217
-
-FROM base-downloader AS base-downloader-linux-arm64
-ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu
-ENV DESCHASHPLUGIN_ARCH=linux-arm64
-ENV DESCHASHPLUGIN_HASH=d48c3e5aede77bd9cb72d78689ce12c0327f624435cb0496b3eacb92df416363
-
-FROM base-downloader AS base-downloader-linux-arm
-ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf
-ENV DESCHASHPLUGIN_ARCH=linux-arm
-ENV DESCHASHPLUGIN_HASH=f7df336c72dd1674bd18ff23862a410b6a9691a3e13752264dcffa0950e21c74
-
-FROM base-downloader-${TARGETOS}-${TARGETARCH} AS downloader
+# This dockerfile is meant to compile a core-lightning x64 image
+# It is using multi stage build:
+# * downloader: Download litecoin/bitcoin and qemu binaries needed for core-lightning
+# * builder: Compile core-lightning dependencies, then core-lightning itself with static linking
+# * final: Copy the binaries required at runtime
+# The resulting image uploaded to dockerhub will only contain what is needed for runtime.
+# From the root of the repository, run "docker build -t yourimage:yourtag ."
+FROM debian:bullseye-slim as downloader
 
 RUN set -ex \
 	&& apt-get update \
@@ -37,9 +13,12 @@ RUN set -ex \
 
 WORKDIR /opt
 
+RUN wget -qO /tini "https://github.com/krallin/tini/releases/download/v0.18.0/tini" \
+    && echo "12d20136605531b09a2c2dac02ccee85e1b874eb322ef6baf7561cd93f93c855 /tini" | sha256sum -c - \
+    && chmod +x /tini
 
-ENV BITCOIN_VERSION=27.1
-ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
+ARG BITCOIN_VERSION=22.0
+ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
 ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/$BITCOIN_TARBALL
 ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS
 
@@ -49,48 +28,42 @@ RUN mkdir /opt/bitcoin && cd /opt/bitcoin \
     && grep $BITCOIN_TARBALL bitcoin | tee SHA256SUMS \
     && sha256sum -c SHA256SUMS \
     && BD=bitcoin-$BITCOIN_VERSION/bin \
-    && tar -xzvf $BITCOIN_TARBALL $BD/ --strip-components=1 \
+    && tar -xzvf $BITCOIN_TARBALL $BD/bitcoin-cli --strip-components=1 \
     && rm $BITCOIN_TARBALL
 
 ENV LITECOIN_VERSION 0.16.3
-ENV LITECOIN_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
+ENV LITECOIN_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-x86_64-linux-gnu.tar.gz
+ENV LITECOIN_SHA256 686d99d1746528648c2c54a1363d046436fd172beadaceea80bdc93043805994
 
 # install litecoin binaries
 RUN mkdir /opt/litecoin && cd /opt/litecoin \
     && wget -qO litecoin.tar.gz "$LITECOIN_URL" \
-    && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \
+    && echo "$LITECOIN_SHA256  litecoin.tar.gz" | sha256sum -c - \
+    && BD=litecoin-$LITECOIN_VERSION/bin \
+    && tar -xzvf litecoin.tar.gz $BD/litecoin-cli --strip-components=1 --exclude=*-qt \
     && rm litecoin.tar.gz
 
-ENV DESCHASHPLUGIN_URL https://github.com/nbd-wtf/invoicewithdescriptionhash/releases/download/v1.4/invoicewithdescriptionhash-v1.4-${DESCHASHPLUGIN_ARCH}.tar.gz
-ENV DESCHASHPLUGIN_SHA256 ${DESCHASHPLUGIN_HASH}
-RUN mkdir /opt/deschashplugin && cd /opt/deschashplugin \
-    && wget -qO invoicewithdescriptionhash.tar.gz "$DESCHASHPLUGIN_URL" \
-    && echo "$DESCHASHPLUGIN_SHA256  invoicewithdescriptionhash.tar.gz" | sha256sum -c - \
-    && tar -xzvf invoicewithdescriptionhash.tar.gz && rm invoicewithdescriptionhash.tar.gz \
-    && chmod a+x invoicewithdescriptionhash
+FROM debian:bullseye-slim as builder
 
-FROM --platform=${DEFAULT_TARGETPLATFORM} ${BASE_DISTRO} AS base-builder
+ENV LIGHTNINGD_VERSION=master
 RUN apt-get update -qq && \
     apt-get install -qq -y --no-install-recommends \
         autoconf \
         automake \
-        bison \
         build-essential \
         ca-certificates \
         curl \
         dirmngr \
-        flex \
         gettext \
         git \
         gnupg \
-        jq \
-        libicu-dev \
+        libpq-dev \
         libtool \
         libffi-dev \
         pkg-config \
         libssl-dev \
         protobuf-compiler \
-        python3 \
+        python3.9 \
         python3-dev \
         python3-mako \
         python3-pip \
@@ -99,246 +72,89 @@ RUN apt-get update -qq && \
         libev-dev \
         libevent-dev \
         qemu-user-static \
-        wget \
-        unzip \
-        tclsh
+        wget
 
-ENV PATH="/root/.local/bin:$PATH" \
-    PYTHON_VERSION=3 \
-    POETRY_VERSION=2.0.1
-RUN curl -sSL https://install.python-poetry.org | python3 - && \
-    poetry self add poetry-plugin-export
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install --upgrade pip setuptools wheel
+RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz \
+    && tar xvf zlib-1.2.13.tar.gz \
+    && cd zlib-1.2.13 \
+    && ./configure \
+    && make \
+    && make install && cd .. && \
+    rm zlib-1.2.13.tar.gz && \
+    rm -rf zlib-1.2.13
 
-RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz -O zlib.tar.gz && \
-    wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip -O sqlite.zip && \
-    wget -q https://ftp.postgresql.org/pub/source/v17.1/postgresql-17.1.tar.gz -O postgres.tar.gz
+RUN apt-get install -y --no-install-recommends unzip tclsh \
+    && wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip \
+    && unzip sqlite-src-3290000.zip \
+    && cd sqlite-src-3290000 \
+    && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension \
+    && make \
+    && make install && cd .. && rm sqlite-src-3290000.zip && rm -rf sqlite-src-3290000
+
+USER root
+RUN wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz \
+    && tar xvf gmp-6.1.2.tar.xz \
+    && cd gmp-6.1.2 \
+    && ./configure --disable-assembly \
+    && make \
+    && make install && cd .. && rm gmp-6.1.2.tar.xz && rm -rf gmp-6.1.2
+
+ENV RUST_PROFILE=release
+ENV PATH=$PATH:/root/.cargo/bin/
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+RUN rustup toolchain install stable --component rustfmt --allow-downgrade
 
 WORKDIR /opt/lightningd
 COPY . /tmp/lightning
+
 RUN git clone --recursive /tmp/lightning . && \
     git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD)
 
-# Do not build python plugins (wss-proxy) here, python doesn't support cross compilation.
-RUN sed -i '/^wss-proxy/d' pyproject.toml && \
-    poetry lock && \
-    poetry export -o requirements.txt --without-hashes
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install -r requirements.txt && \
-    pip3 cache purge
-WORKDIR /
-
-FROM base-builder AS base-builder-linux-amd64
-
-ENV POSTGRES_CONFIG="--without-readline" \
-    PG_CONFIG=/usr/local/pgsql/bin/pg_config
-
-FROM base-builder AS base-builder-linux-arm64
-ENV target_host=aarch64-linux-gnu \
-    target_host_rust=aarch64-unknown-linux-gnu \
-    target_host_qemu=qemu-aarch64-static
-
-RUN apt-get install -qq -y --no-install-recommends \
-        libc6-arm64-cross \
-        gcc-${target_host} \
-        g++-${target_host}
-
-ENV AR=${target_host}-ar \
-    AS=${target_host}-as \
-    CC=${target_host}-gcc \
-    CXX=${target_host}-g++ \
-    LD=${target_host}-ld \
-    STRIP=${target_host}-strip \
-    QEMU_LD_PREFIX=/usr/${target_host} \
-    HOST=${target_host} \
-    TARGET=${target_host_rust} \
-    RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
-    PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
-
-ENV ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
-    SQLITE_CONFIG="--host=${target_host} --prefix=${QEMU_LD_PREFIX}" \
-    POSTGRES_CONFIG="--without-readline --prefix=${QEMU_LD_PREFIX}" \
-    PG_CONFIG="${QEMU_LD_PREFIX}/bin/pg_config"
-
-FROM base-builder AS base-builder-linux-arm
-
-ENV target_host=arm-linux-gnueabihf \
-    target_host_rust=armv7-unknown-linux-gnueabihf \
-    target_host_qemu=qemu-arm-static
-
-RUN apt-get install -qq -y --no-install-recommends \
-        libc6-armhf-cross \
-        gcc-${target_host} \
-        g++-${target_host}
-
-ENV AR=${target_host}-ar \
-    AS=${target_host}-as \
-    CC=${target_host}-gcc \
-    CXX=${target_host}-g++ \
-    LD=${target_host}-ld \
-    STRIP=${target_host}-strip \
-    QEMU_LD_PREFIX=/usr/${target_host} \
-    HOST=${target_host} \
-    TARGET=${target_host_rust} \
-    RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
-    PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
-
-ENV ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
-    SQLITE_CONFIG="--host=${target_host} --prefix=${QEMU_LD_PREFIX}" \
-    POSTGRES_CONFIG="--without-readline --prefix=${QEMU_LD_PREFIX}" \
-    PG_CONFIG="${QEMU_LD_PREFIX}/bin/pg_config"
-
-FROM base-builder-${TARGETOS}-${TARGETARCH} AS builder
-
-ENV LIGHTNINGD_VERSION=master
-
-RUN mkdir zlib && tar xvf zlib.tar.gz -C zlib --strip-components=1 \
-    && cd zlib \
-    && ./configure ${ZLIB_CONFIG} \
-    && make \
-    && make install && cd .. && \
-    rm zlib.tar.gz && \
-    rm -rf zlib
-
-RUN unzip sqlite.zip \
-    && cd sqlite-* \
-    && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension ${SQLITE_CONFIG} \
-    && make \
-    && make install && cd .. && rm sqlite.zip && rm -rf sqlite-*
-
-RUN mkdir postgres && tar xvf postgres.tar.gz -C postgres --strip-components=1 \
-    && cd postgres \
-    && ./configure ${POSTGRES_CONFIG} \
-    && cd src/include \
-    && make install \
-    && cd ../interfaces/libpq \
-    && make install \
-    && cd ../../bin/pg_config \
-    && make install \
-    && cd ../../../../ && \
-    rm postgres.tar.gz && \
-    rm -rf postgres && \
-    ldconfig "$(${PG_CONFIG} --libdir)"
-
-# Save libpq to a specific location to copy it into the final image.
-RUN mkdir /var/libpq && cp -a "$(${PG_CONFIG} --libdir)"/libpq.* /var/libpq
-
-ENV RUST_PROFILE=release \
-    PATH="/root/.cargo/bin:/root/.local/bin:$PATH"
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
-RUN rustup toolchain install stable --component rustfmt --allow-downgrade
-
-COPY --from=downloader /usr/bin/${target_host_qemu} /usr/bin/${target_host_qemu}
-WORKDIR /opt/lightningd
-
-# If cross-compiling, need to tell it to cargo.
-RUN ( ! [ -n "${target_host}" ] ) || \
-    (mkdir -p .cargo && echo "[target.${target_host_rust}]\nlinker = \"${target_host}-gcc\"" > .cargo/config)
-
-# Weird errors with cargo for cln-grpc on arm7 https://github.com/ElementsProject/lightning/issues/6596
-RUN ( ! [ "${target_host}" = "arm-linux-gnueabihf" ] ) || \
-    (sed -i '/documentation = "https:\/\/docs.rs\/cln-grpc"/a include = ["**\/*.*"]' cln-grpc/Cargo.toml)
-
-# Ensure that the desired grpcio-tools & protobuf versions are installed
-# https://github.com/ElementsProject/lightning/pull/7376#issuecomment-2161102381
-RUN poetry lock && poetry install && \
-    poetry self add poetry-plugin-export
-
-# Ensure that git differences are removed before making bineries, to avoid `-modded` suffix
-# poetry.lock changed due to pyln-client, pyln-proto and pyln-testing version updates
-# pyproject.toml was updated to exclude wss-proxy plugins in base-builder stage
-RUN git reset --hard HEAD
-
-RUN ./configure --prefix=/tmp/lightning_install --enable-static && poetry run make install
-
-# Export the requirements for the plugins so we can install them in builder-python stage
-WORKDIR /opt/lightningd/plugins/wss-proxy
-RUN poetry lock && poetry export -o requirements.txt --without-hashes
-WORKDIR /opt/lightningd
-RUN echo 'RUSTUP_INSTALL_OPTS="${RUSTUP_INSTALL_OPTS}"' > /tmp/rustup_install_opts.txt
-
-# We need to build python plugins on the target's arch because python doesn't support cross build
-FROM ${BASE_DISTRO} AS builder-python
-RUN apt-get update -qq && \
-    apt-get install -qq -y --no-install-recommends \
-        git \
-        curl \
-        libtool \
-        pkg-config \
-        autoconf \
-        automake \
-        build-essential \
-        libffi-dev \
-        libssl-dev \
-        python3 \
-        python3-dev \
-        python3-pip \
-        python3-venv && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
+ARG DEVELOPER=1
 ENV PYTHON_VERSION=3
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install --upgrade pip setuptools wheel
+RUN curl -sSL https://install.python-poetry.org | python3 -
 
-# Copy rustup_install_opts.txt file from builder
-COPY --from=builder /tmp/rustup_install_opts.txt /tmp/rustup_install_opts.txt
-# Setup ENV $RUSTUP_INSTALL_OPTS for this stage
-RUN export $(cat /tmp/rustup_install_opts.txt)
-ENV PATH="/root/.cargo/bin:/root/.venvs/cln/bin:$PATH"
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1
 
-WORKDIR /opt/lightningd/plugins/wss-proxy
-COPY --from=builder /opt/lightningd/plugins/wss-proxy/requirements.txt .
-RUN pip3 install -r requirements.txt
-RUN pip3 cache purge
+RUN pip3 install --upgrade pip setuptools wheel
+RUN pip3 wheel cryptography
+RUN pip3 install grpcio-tools
 
-WORKDIR /opt/lightningd
+RUN /root/.local/bin/poetry install
 
-FROM ${BASE_DISTRO} AS final
+RUN ./configure --prefix=/tmp/lightning_install --enable-static && \
+    make DEVELOPER=${DEVELOPER} && \
+    /root/.local/bin/poetry run make install
+
+RUN pip3 install -r plugins/clnrest/requirements.txt
+RUN pip3 install ./contrib/pyln-client
+
+FROM debian:bullseye-slim as final
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-      tini \
       socat \
       inotify-tools \
-      jq \
-      python3 \
-      python3-pip && \
-    apt-get clean && \
+      python3.9 \
+      python3-pip \
+      qemu-user-static \
+      libpq5 && \
     rm -rf /var/lib/apt/lists/*
 
-ENV LIGHTNINGD_DATA=/root/.lightning \
-    LIGHTNINGD_RPC_PORT=9835 \
-    LIGHTNINGD_PORT=9735 \
-    LIGHTNINGD_NETWORK=bitcoin
+ENV LIGHTNINGD_DATA=/root/.lightning
+ENV LIGHTNINGD_RPC_PORT=9835
+ENV LIGHTNINGD_PORT=9735
+ENV LIGHTNINGD_NETWORK=bitcoin
 
 RUN mkdir $LIGHTNINGD_DATA && \
-    mkdir /etc/bundledplugins && \
-    mkdir $LIGHTNINGD_DATA/plugins && \
     touch $LIGHTNINGD_DATA/config
 VOLUME [ "/root/.lightning" ]
 
-# Take libpq directly from builder.
-RUN mkdir /var/libpq && mkdir -p /usr/local/pgsql/lib
-RUN --mount=type=bind,from=builder,source=/var/libpq,target=/var/libpq,rw \
-    cp -a /var/libpq/libpq.* /usr/local/pgsql/lib && \
-    echo "/usr/local/pgsql/lib" > /etc/ld.so.conf.d/libpq.conf && \
-    ldconfig
-
 COPY --from=builder /tmp/lightning_install/ /usr/local/
-COPY --from=builder-python /root/.venvs/cln/lib/python3.11/site-packages /usr/local/lib/python3.11/dist-packages/
+COPY --from=builder /usr/local/lib/python3.9/dist-packages/ /usr/local/lib/python3.9/dist-packages/
 COPY --from=downloader /opt/bitcoin/bin /usr/bin
 COPY --from=downloader /opt/litecoin/bin /usr/bin
-COPY --from=downloader /opt/deschashplugin $LIGHTNINGD_DATA/plugins
-COPY --from=downloader /opt/deschashplugin /etc/bundledplugins
+COPY --from=downloader "/tini" /usr/bin/tini
 COPY tools/docker-entrypoint.sh entrypoint.sh
 
 EXPOSE 9735 9835

LICENSE

@@ -1,7 +1,7 @@
 Note: the modules in the ccan/ directory have their own licenses, but
 the rest of the code is covered by the following (BSD-MIT) license:
 
-	Copyright Rusty Russell (Blockstream) 2015-2024.
+	Copyright Rusty Russell (Blockstream) 2015-2023.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,10 +1,7 @@
 #! /usr/bin/make
 
 # Extract version from git, or if we're from a zipfile, use dirname
-VERSION=$(shell git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|v\1|gp')
-
-# Next release.
-CLN_NEXT_VERSION := v25.05
+VERSION=$(shell git describe --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|\1|gp')
 
 # --quiet / -s means quiet, dammit!
 ifeq ($(findstring s,$(word 1, $(MAKEFLAGS))),s)
@@ -26,7 +23,7 @@ CCANDIR := ccan
 
 # Where we keep the BOLT RFCs
 BOLTDIR := ../bolts/
-DEFAULT_BOLTVERSION := ccfa38ed4f592c3711156bb4ded77f44ec01101d
+DEFAULT_BOLTVERSION := c4c5a8e5fb30b1b99fa5bb0aba7d0b6b4c831ee5
 # Can be overridden on cmdline.
 BOLTVERSION := $(DEFAULT_BOLTVERSION)
 
@@ -36,7 +33,7 @@ SORT=LC_ALL=C sort
 
 
 ifeq ($V,1)
-VERBOSE = $(ECHO) '$(subst ','\'',$(2))'; $(2)
+VERBOSE = $(ECHO) '$(2)'; $(2)
 else
 VERBOSE = $(ECHO) $(1); $(2)
 endif
@@ -72,7 +69,7 @@ endif
 
 # (method=thread to support xdist)
 PYTEST_OPTS := -v -p no:logging $(PYTEST_OPTS)
-MY_CHECK_PYTHONPATH=$${PYTHONPATH}$${PYTHONPATH:+:}$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7:$(shell pwd)/contrib/pyln-grpc-proto
+MY_CHECK_PYTHONPATH=$${PYTHONPATH}$${PYTHONPATH:+:}$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/external/lnprototest:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7
 # Collect generated python files to be excluded from lint checks
 PYTHON_GENERATED= \
 	contrib/pyln-grpc-proto/pyln/grpc/primitives_pb2.py \
@@ -80,6 +77,10 @@ PYTHON_GENERATED= \
 	contrib/pyln-grpc-proto/pyln/grpc/node_pb2.py \
 	contrib/pyln-testing/pyln/testing/grpc2py.py
 
+# Options to pass to cppcheck. Mostly used to exclude files that are
+# generated with external tools that we don't have control over
+CPPCHECK_OPTS=-q --language=c --std=c11 --error-exitcode=1 --suppressions-list=.cppcheck-suppress --inline-suppr
+
 # This is where we add new features as bitcoin adds them.
 FEATURES :=
 
@@ -228,24 +229,10 @@ ALL_TEST_PROGRAMS :=
 ALL_TEST_GEN :=
 ALL_FUZZ_TARGETS :=
 ALL_C_SOURCES :=
-ALL_C_HEADERS :=
+ALL_C_HEADERS := header_versions_gen.h version_gen.h
 # Extra (non C) targets that should be built by default.
 DEFAULT_TARGETS :=
 
-# Installation directories
-exec_prefix = $(PREFIX)
-bindir = $(exec_prefix)/bin
-libexecdir = $(exec_prefix)/libexec
-pkglibexecdir = $(libexecdir)/$(PKGNAME)
-plugindir = $(pkglibexecdir)/plugins
-datadir = $(PREFIX)/share
-docdir = $(datadir)/doc/$(PKGNAME)
-mandir = $(datadir)/man
-man1dir = $(mandir)/man1
-man5dir = $(mandir)/man5
-man7dir = $(mandir)/man7
-man8dir = $(mandir)/man8
-
 # M1 macos machines with homebrew will install the native libraries in
 # /opt/homebrew instead of /usr/local, most likely because they
 # emulate x86_64 compatibility via Rosetta, and wanting to keep the
@@ -262,8 +249,8 @@ CPATH := /usr/local/include
 LIBRARY_PATH := /usr/local/lib
 endif
 
-CPPFLAGS += -DCLN_NEXT_VERSION="\"$(CLN_NEXT_VERSION)\"" -DPKGLIBEXECDIR="\"$(pkglibexecdir)\"" -DBINDIR="\"$(bindir)\"" -DPLUGINDIR="\"$(plugindir)\"" -DCCAN_TAL_NEVER_RETURN_NULL=1
-CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I$(CPATH) $(SQLITE3_CFLAGS) $(SODIUM_CFLAGS) $(POSTGRES_INCLUDE) $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) $(CSANFLAGS)
+CPPFLAGS += -DBINTOPKGLIBEXECDIR="\"$(shell sh tools/rel.sh $(bindir) $(pkglibexecdir))\""
+CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I$(CPATH) $(SQLITE3_CFLAGS) $(POSTGRES_INCLUDE) $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) $(CSANFLAGS) -DBUILD_ELEMENTS=1
 
 # If CFLAGS is already set in the environment of make (to whatever value, it
 # does not matter) then it would export it to subprocesses with the above value
@@ -281,9 +268,9 @@ ifeq ($(STATIC),1)
 # For MacOS, Jacob Rapoport <jacob@rumblemonkey.com> changed this to:
 #  -L/usr/local/lib -lsqlite3 -lz -Wl,-lm -lpthread -ldl $(COVFLAGS)
 # But that doesn't static link.
-LDLIBS = -L$(CPATH) -Wl,-dn $(SQLITE3_LDLIBS) -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
+LDLIBS = -L$(CPATH) -Wl,-dn $(SQLITE3_LDLIBS) -lz -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
 else
-LDLIBS = -L$(CPATH) -lm $(SQLITE3_LDLIBS) $(COVFLAGS)
+LDLIBS = -L$(CPATH) -lm $(SQLITE3_LDLIBS) -lz $(COVFLAGS)
 endif
 
 # If we have the postgres client library we need to link against it as well
@@ -302,11 +289,13 @@ show-flags: config.vars
 	@$(ECHO) "CC: $(CC) $(CFLAGS) -c -o"
 	@$(ECHO) "LD: $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) -o"
 
-# We will re-generate, but we won't generate for the first time!
-ccan/config.h config.vars &: configure ccan/tools/configurator/configurator.c
-	@if [ ! -f config.vars ]; then echo 'File config.vars not found: you must run ./configure before running make.' >&2; exit 1; fi
+ccan/config.h: config.vars configure ccan/tools/configurator/configurator.c
 	./configure --reconfigure
 
+config.vars:
+	@echo 'File config.vars not found: you must run ./configure before running make.' >&2
+	@exit 1
+
 %.o: %.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 
@@ -366,26 +355,20 @@ include connectd/Makefile
 include lightningd/Makefile
 include cli/Makefile
 include doc/Makefile
-include contrib/msggen/Makefile
 include devtools/Makefile
 include tools/Makefile
-ifneq ($(RUST),0)
-include cln-rpc/Makefile
-include cln-grpc/Makefile
-endif
 include plugins/Makefile
 include tests/plugins/Makefile
 
 ifneq ($(FUZZING),0)
 	include tests/fuzz/Makefile
 endif
+ifneq ($(RUST),0)
+	include cln-rpc/Makefile
+	include cln-grpc/Makefile
 
-ifneq ($V,1)
-MSGGEN_ARGS := -s
-endif
-
-$(MSGGEN_GENALL)&: contrib/msggen/msggen/schema.json
-	@$(call VERBOSE, "msggen $@", PYTHONPATH=contrib/msggen $(PYTHON) contrib/msggen/msggen/__main__.py $(MSGGEN_ARGS) generate)
+$(MSGGEN_GENALL)&: doc/schemas/*.request.json doc/schemas/*.schema.json
+	PYTHONPATH=contrib/msggen python3 contrib/msggen/msggen/__main__.py
 
 # The compiler assumes that the proto files are in the same
 # directory structure as the generated files will be. Since we
@@ -400,11 +383,12 @@ GRPC_GEN = \
 
 ALL_TEST_GEN += $(GRPC_GEN)
 
-$(GRPC_GEN) &: cln-grpc/proto/node.proto cln-grpc/proto/primitives.proto
-	$(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/node.proto --python_out=$(GRPC_PATH)/ --grpc_python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
-	$(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/primitives.proto --python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
+$(GRPC_GEN)&: cln-grpc/proto/node.proto cln-grpc/proto/primitives.proto
+	python -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/node.proto --python_out=$(GRPC_PATH)/ --grpc_python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
+	python -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/primitives.proto --python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
 	find $(GRPC_DIR)/ -type f -name "*.py" -print0 | xargs -0 sed -i'.bak' -e 's/^import \(.*\)_pb2 as .*__pb2/from pyln.grpc import \1_pb2 as \1__pb2/g'
-	find $(GRPC_DIR)/ -type f -name "*.py.bak" -print0 | xargs -0 rm -f
+	find $(GRPC_DIR)/ -type f -name "*.py.bak" -delete
+endif
 
 # We make pretty much everything depend on these.
 ALL_GEN_HEADERS := $(filter %gen.h,$(ALL_C_HEADERS))
@@ -437,7 +421,7 @@ mkdocs.yml: $(MANPAGES:=.md)
 	  find doc -maxdepth 1 -name '*\.[0-9]\.md' | \
 	  cut -b 5- | LC_ALL=C sort | \
 	  sed 's/\(.*\)\.\(.*\).*\.md/- "\1": "\1.\2.md"/' | \
-	  $(PYTHON) devtools/blockreplace.py mkdocs.yml manpages --language=yml --indent "          " \
+	  python3 devtools/blockreplace.py mkdocs.yml manpages --language=yml --indent "          " \
 	)
 
 
@@ -466,22 +450,28 @@ else
 PYTEST_OPTS += -x
 endif
 
-# Allow for targeting specific tests by setting the PYTEST_TESTS environment variable.
-ifeq ($(PYTEST_TESTS),)
-PYTEST_TESTS = "tests/"
-endif
-
 check-units:
 
-check: check-units installcheck pytest
+check: check-units installcheck check-protos pytest
+
+check-protos: $(ALL_PROGRAMS)
+ifeq ($(PYTEST),)
+	@echo "py.test is required to run the protocol tests, please install using 'pip3 install -r requirements.txt', and rerun 'configure'."; false
+else
+ifeq ($(DEVELOPER),1)
+	@(cd external/lnprototest && PYTHONPATH=$(MY_CHECK_PYTHONPATH) LIGHTNING_SRC=../.. $(PYTEST) --runner lnprototest.clightning.Runner $(PYTEST_OPTS))
+else
+	@echo "lnprototest target requires DEVELOPER=1, skipping"
+endif
+endif
 
 pytest: $(ALL_PROGRAMS) $(DEFAULT_TARGETS) $(ALL_TEST_PROGRAMS) $(ALL_TEST_GEN)
 ifeq ($(PYTEST),)
 	@echo "py.test is required to run the integration tests, please install using 'pip3 install -r requirements.txt', and rerun 'configure'."
 	exit 1
 else
-# Explicitly hand VALGRIND so you can override on make cmd line.
-	PYTHONPATH=$(MY_CHECK_PYTHONPATH) TEST_DEBUG=1 VALGRIND=$(VALGRIND) $(PYTEST) $(PYTEST_TESTS) $(PYTEST_OPTS)
+# Explicitly hand DEVELOPER and VALGRIND so you can override on make cmd line.
+	PYTHONPATH=$(MY_CHECK_PYTHONPATH) TEST_DEBUG=1 DEVELOPER=$(DEVELOPER) VALGRIND=$(VALGRIND) $(PYTEST) tests/ $(PYTEST_OPTS)
 endif
 
 check-fuzz: $(ALL_FUZZ_TARGETS)
@@ -559,8 +549,15 @@ check-pytest-pyln-proto:
 check-includes: check-src-includes check-hdr-includes
 	@tools/check-includes.sh
 
+# cppcheck gets confused by list_for_each(head, i, list): thinks i is uninit.
+.cppcheck-suppress:
+	@git ls-files -- "*.c" "*.h" | grep -vE '^(ccan|contrib)/' | xargs grep -n '_for_each' | sed 's/\([^:]*:.*\):.*/uninitvar:\1/' > $@
+
+check-cppcheck: .cppcheck-suppress
+	@trap 'rm -f .cppcheck-suppress' 0; git ls-files -- "*.c" "*.h" | grep -vE '^ccan/' | xargs cppcheck  ${CPPCHECK_OPTS}
+
 check-shellcheck:
-	@git ls-files -z -- "*.sh" | xargs -0 shellcheck -f gcc
+	@git ls-files -- "*.sh" | xargs shellcheck -f gcc
 
 check-setup_locale:
 	@tools/check-setup_locale.sh
@@ -575,29 +572,12 @@ check-discouraged-functions:
 # since it risks overflow.
 check-amount-access:
 	@! (git grep -nE "(->|\.)(milli)?satoshis" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" | grep -v '/* Raw:')
-	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" | grep -vE "sizeof.struct amount_(m)?sat."
-
-repeat-doc-examples:
-	@for i in $$(seq 1 $(n)); do \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-		echo "Iteration $$i" >> tests/autogenerate-examples-repeat.log; \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-		VALGRIND=0 TIMEOUT=40 TEST_DEBUG=1 GENERATE_EXAMPLES=1 pytest -vvv tests/autogenerate-rpc-examples.py; \
-		git diff >> tests/autogenerate-examples-repeat.log; \
-		git reset --hard; \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-	done
-
-update-doc-examples:
-	TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 $(PYTEST) $(PYTEST_OPTS) --timeout=1200 tests/autogenerate-rpc-examples.py && $(MAKE) $(MSGGEN_GEN_ALL)
-
-check-doc-examples: update-doc-examples
-	git diff --exit-code HEAD
+	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*"
 
 # For those without working cppcheck
 check-source-no-cppcheck: check-makefile check-source-bolt check-whitespace check-spelling check-python check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access
 
-check-source: check-source-no-cppcheck
+check-source: check-source-no-cppcheck check-cppcheck
 
 full-check: check check-source
 
@@ -617,8 +597,8 @@ CHECK_GEN_ALL = \
 	$(PYTHON_GENERATED) \
 	$(ALL_GEN_HEADERS) \
 	$(ALL_GEN_SOURCES) \
-	$(MSGGEN_GEN_ALL) \
 	wallet/statements_gettextgen.po \
+	.msggen.json \
 	doc/index.rst
 
 gen:  $(CHECK_GEN_ALL)
@@ -641,10 +621,10 @@ ncc: ${TARGET_DIR}/libwally-core-build/src/libwallycore.la
 
 # Ignore test/ directories.
 TAGS:
-	$(RM) TAGS; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 etags --append
+	$(RM) TAGS; find * -name test -type d -prune -o -name '*.[ch]' -print -o -name '*.py' -print | xargs etags --append
 
 tags:
-	$(RM) tags; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 ctags --append
+	$(RM) tags; find * -name test -type d -prune -o -name '*.[ch]' -print -o -name '*.py' -print | xargs ctags --append
 
 ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS)
 
@@ -665,7 +645,7 @@ version_gen.h: $(FORCE)
 endif
 
 # That forces this rule to be run every time, too.
-header_versions_gen.h: tools/headerversions $(FORCE)
+header_versions_gen.h: tools/headerversions
 	@tools/headerversions $@
 
 # We make a static library, this way linker can discard unused parts.
@@ -683,7 +663,7 @@ $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): %: %.o
 # uses some ccan modules internally).  We want to rely on -lwallycore etc.
 # (as per EXTERNAL_LDLIBS) so we filter them out here.
 $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS):
-	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a $($(@)_LDLIBS) -o $@)
+	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a -o $@)
 
 # We special case the fuzzing target binaries, as they need to link against libfuzzer,
 # which brings its own main().
@@ -714,23 +694,20 @@ update-ccan:
 
 # Now ALL_PROGRAMS is fully populated, we can expand it.
 all-programs: $(ALL_PROGRAMS)
-all-fuzz-programs: $(ALL_FUZZ_TARGETS)
 all-test-programs: $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS)
 default-targets: $(DEFAULT_TARGETS)
 
 distclean: clean
 	$(RM) ccan/config.h config.vars
+	$(RM) $(PYTHON_GENERATED)
 
 maintainer-clean: distclean
 	@echo 'This command is intended for maintainers to use; it'
 	@echo 'deletes files that may need special tools to rebuild.'
-	$(RM) $(PYTHON_GENERATED)
 
 # We used to have gen_ files, now we have _gen files.
-# We used to generate doc/schemas/lightning-sql.json.
 obsclean:
 	$(RM) gen_*.h */gen_*.[ch] */*/gen_*.[ch]
-	$(RM) doc/schemas/lightning-sql.json
 
 clean: obsclean
 	$(RM) libccan.a $(CCAN_OBJS) $(CDUMP_OBJS) $(ALL_OBJS)
@@ -738,7 +715,6 @@ clean: obsclean
 	$(RM) $(ALL_PROGRAMS)
 	$(RM) $(ALL_TEST_PROGRAMS)
 	$(RM) $(ALL_FUZZ_TARGETS)
-	$(RM) $(MSGGEN_GEN_ALL)
 	$(RM) ccan/tools/configurator/configurator
 	$(RM) ccan/ccan/cdump/tools/cdump-enumstr.o
 	find . -name '*gcda' -delete
@@ -748,9 +724,7 @@ clean: obsclean
 
 
 PYLNS=client proto testing
-# See doc/contribute-to-core-lightning/contributor-workflow.md
-update-versions: update-pyln-versions update-wss-proxy-version update-poetry-lock update-dot-version update-doc-examples
-
+# See doc/MAKING-RELEASES.md
 update-pyln-versions: $(PYLNS:%=update-pyln-version-%)
 
 update-pyln-version-%:
@@ -762,30 +736,35 @@ pyln-release:  $(PYLNS:%=pyln-release-%)
 pyln-release-%:
 	cd contrib/pyln-$* && $(MAKE) prod-release
 
-update-wss-proxy-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	cd plugins/wss-proxy && $(MAKE) upgrade-version
-
-update-poetry-lock:
-	poetry update wss-proxy pyln-client pyln-proto pyln-testing update-reckless-version
-
-update-reckless-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	@sed -i "s/__VERSION__ = '\([.-z]*\)'/__VERSION__ = '$(NEW_VERSION)'/" tools/reckless
-
-update-dot-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	echo $(NEW_VERSION) > .version
-
+# These must both be enabled for update-mocks
+ifeq ($(DEVELOPER),1)
 update-mocks: $(ALL_TEST_PROGRAMS:%=update-mocks/%.c)
+else
+update-mocks:
+	@echo Need DEVELOPER=1 to regenerate mocks >&2; exit 1
+endif
 
 $(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) libccan.a ccan/ccan/cdump/tools/cdump-enumstr config.vars
 
-update-mocks/%: % $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES)
+update-mocks/%: %
 	@MAKE=$(MAKE) tools/update-mocks.sh "$*" $(SUPPRESS_OUTPUT)
 
-unittest/%: % bolt-precheck
-	BOLTDIR=$(LOCAL_BOLTDIR) $(VG) $(VG_TEST_ARGS) $* > /dev/null
+unittest/%: %
+	$(VG) $(VG_TEST_ARGS) $* > /dev/null
+
+# Installation directories
+exec_prefix = $(PREFIX)
+bindir = $(exec_prefix)/bin
+libexecdir = $(exec_prefix)/libexec
+pkglibexecdir = $(libexecdir)/$(PKGNAME)
+plugindir = $(pkglibexecdir)/plugins
+datadir = $(PREFIX)/share
+docdir = $(datadir)/doc/$(PKGNAME)
+mandir = $(datadir)/man
+man1dir = $(mandir)/man1
+man5dir = $(mandir)/man5
+man7dir = $(mandir)/man7
+man8dir = $(mandir)/man8
 
 # Commands
 MKDIR_P = mkdir -p
@@ -819,7 +798,6 @@ install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $
 	@$(NORMAL_INSTALL)
 	$(INSTALL_PROGRAM) $(BIN_PROGRAMS) $(DESTDIR)$(bindir)
 	$(INSTALL_PROGRAM) $(PKGLIBEXEC_PROGRAMS) $(DESTDIR)$(pkglibexecdir)
-	@if [ -d "$(DESTDIR)$(plugindir)/clnrest" ]; then rm -rf $(DESTDIR)$(plugindir)/clnrest; fi
 	[ -z "$(PLUGINS)" ] || $(INSTALL_PROGRAM) $(PLUGINS) $(DESTDIR)$(plugindir)
 	for PY in $(PY_PLUGINS); do DIR=`dirname $$PY`; DST=$(DESTDIR)$(plugindir)/`basename $$DIR`; if [ -d $$DST ]; then rm -rf $$DST; fi; $(INSTALL_PROGRAM) -d $$DIR; cp -a $$DIR $$DST ; done
 
@@ -846,7 +824,6 @@ install: install-program install-data
 TESTBINS = \
 	$(CLN_PLUGIN_EXAMPLES) \
 	tests/plugins/test_libplugin \
-	tests/plugins/channeld_fakenet \
 	tests/plugins/test_selfdisable_after_getmanifest \
 	tools/hsmtool
 
@@ -910,11 +887,8 @@ installcheck: all-programs
 	fi
 	@rm -rf testinstall || true
 
-version:
-	@echo ${VERSION}
-
 .PHONY: installdirs install-program install-data install uninstall \
-	installcheck ncc bin-tarball show-flags version
+	installcheck ncc bin-tarball show-flags
 
 # Make a tarball of opt/clightning/, optionally with label for distribution.
 ifneq ($(VERSION),)

README.md

@@ -19,30 +19,25 @@ Core Lightning (previously c-lightning) is a lightweight, highly customizable an
 
 ## Project Status
 
-[![Continuous Integration][actions-badge]][actions]
-[![Pull Requests Welcome][prs-badge]][prs]
-[![Documentation Status][docs-badge]][docs]
-[![BoL2][bol2-badge]][bol2]
-[![Telegram][telegram-badge]][telegram]
-[![Discord][discord-badge]][discord]
-[![Irc][IRC-badge]][IRC]
+[![Continuous Integration](https://github.com/ElementsProject/lightning/workflows/Continuous%20Integration/badge.svg)][actions]
+[![Pull Requests Welcome][prs]][prs-link]
+[![Irc][IRC]][IRC-link]
+[![Documentation Status](https://readthedocs.org/projects/lightning/badge/?version=docs)][docs]
 
 This implementation has been in production use on the Bitcoin mainnet since early 2018, with the launch of the [Blockstream Store][blockstream-store-blog].
-We recommend getting started by experimenting on `testnet` (`testnet4` or `regtest`), but the implementation is considered stable and can be safely used on mainnet.
-
-## Reach Out to Us
+We recommend getting started by experimenting on `testnet` (or `regtest`), but the implementation is considered stable and can be safely used on mainnet.
 
 Any help testing the implementation, reporting bugs, or helping with outstanding issues is very welcome.
-Don't hesitate to reach out to us on [Build-on-L2][bol2], or on the implementation-specific [mailing list][ml1], or on [CLN Discord][discord], or on [CLN Telegram][telegram], or on IRC at [dev][irc1]/[gen][irc2] channel.
+Don't hesitate to reach out to us on IRC at [#lightning-dev @ libera.chat][irc1], [#c-lightning @ libera.chat][irc2], or on the implementation-specific mailing list [c-lightning@lists.ozlabs.org][ml1], or on the Lightning Network-wide mailing list [lightning-dev@lists.linuxfoundation.org][ml2], or on Discord [core-lightning][discord], or on Telegram [Core Lightning][telegram].
 
 ## Getting Started
 
-Core Lightning only works on Linux and macOS, and requires a locally (or remotely) running `bitcoind` (version 25.0 or above) that is fully caught up with the network you're running on, and relays transactions (ie with `blocksonly=0`).
+Core Lightning only works on Linux and macOS, and requires a locally (or remotely) running `bitcoind` (version 0.16 or above) that is fully caught up with the network you're running on, and relays transactions (ie with `blocksonly=0`).
 Pruning (`prune=n` option in `bitcoin.conf`) is partially supported, see [here](#pruning) for more details.
 
 ### Installation
 
-There are 3 supported installation options:
+There are 4 supported installation options:
 
  - Installation of a pre-compiled binary from the [release page][releases] on GitHub.
  - Using one of the [provided docker images][dockerhub] on the Docker Hub.
@@ -60,6 +55,13 @@ of the `startup_regtest.sh` file for details on how to use it.
 . contrib/startup_regtest.sh
 ```
 
+Note that your local nodeset will be much faster/more responsive if
+you've configured your node to expose the developer options, e.g.
+
+```bash
+./configure --enable-developer
+```
+
 #### Mainnet Option
 To test with real bitcoin,  you will need to have a local `bitcoind` node running:
 
@@ -106,10 +108,13 @@ the lightning network.
 There are also numerous plugins available for Core Lightning which add
 capabilities: in particular there's a collection at: https://github.com/lightningd/plugins
 
+Including [helpme][helpme-github] which guides you through setting up
+your first channels and customizing your node.
+
 For a less reckless experience, you can encrypt the HD wallet seed:
  see [HD wallet encryption](#hd-wallet-encryption).
 
-You can also chat to other users at Discord [core-lightning][discord];
+You can also chat to other users at [#c-lightning @ libera.chat][irc2];
 we are always happy to help you get started!
 
 
@@ -125,11 +130,11 @@ lightning-cli newaddr
 
 `lightningd` will register the funds once the transaction is confirmed.
 
-Alternatively you can generate a taproot address should your source of funds support it:
+You may need to generate a p2sh-segwit address if the faucet does not support bech32:
 
 ```bash
-# Return a taproot address
-lightning-cli newaddr p2tr
+# Return a p2sh-segwit address
+lightning-cli newaddr p2sh-segwit
 ```
 
 Confirm `lightningd` got funds by:
@@ -206,27 +211,23 @@ If you encrypt your `hsm_secret`, you will have to pass the `--encrypted-hsm` st
 ### Developers
 
 Developers wishing to contribute should start with the developer guide [here](doc/contribute-to-core-lightning/coding-style-guidelines.md).
+You should also configure with `--enable-developer` to get additional checks and options.
 
 [blockstream-store-blog]: https://blockstream.com/2018/01/16/en-lightning-charge/
 [std]: https://github.com/lightning/bolts
-[prs-badge]: https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat
-[prs]: http://makeapullrequest.com
-[bol2-badge]: https://badgen.net/badge/BoL2/chat/blue
-[bol2]: https://community.corelightning.org
-[ml1]: https://lists.ozlabs.org/listinfo/c-lightning
-[discord-badge]: https://badgen.net/badge/Discord/chat/blue
-[discord]: https://discord.gg/mE9s4rc5un
-[telegram-badge]: https://badgen.net/badge/Telegram/chat/blue
-[telegram]: https://t.me/lightningd
-[IRC-badge]: https://img.shields.io/badge/IRC-chat-blue.svg
-[IRC]: https://web.libera.chat/#c-lightning
+[prs]: https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat
+[prs-link]: http://makeapullrequest.com
+[IRC]: https://img.shields.io/badge/chat-on%20libera-brightgreen.svg
+[IRC-link]: https://web.libera.chat/#c-lightning
 [irc1]: https://web.libera.chat/#lightning-dev
 [irc2]: https://web.libera.chat/#c-lightning
-[docs-badge]: https://readthedocs.org/projects/lightning/badge/?version=docs
+[ml1]: https://lists.ozlabs.org/listinfo/c-lightning
+[ml2]: https://lists.linuxfoundation.org/mailman/listinfo/lightning-dev
+[discord]: https://discord.gg/mE9s4rc5un
+[telegram]: https://t.me/lightningd
 [docs]: https://docs.corelightning.org/docs
 [releases]: https://github.com/ElementsProject/lightning/releases
 [dockerhub]: https://hub.docker.com/r/elementsproject/lightningd/
 [jsonrpcspec]: https://www.jsonrpc.org/specification
 [helpme-github]: https://github.com/lightningd/plugins/tree/master/helpme
-[actions-badge]: https://github.com/ElementsProject/lightning/workflows/Continuous%20Integration/badge.svg
 [actions]: https://github.com/ElementsProject/lightning/actions

SECURITY.md

@@ -6,30 +6,19 @@ We have a 3 month release cycle, and the last two versions are supported.
 
 ## Reporting a Vulnerability
 
-To report security vulnerabilities, please send an email to one of the following addresses:
-- `rusty@rustcorp.com.au`
-- `security@blockstream.com`
-
-Note: These email addresses are exclusively for vulnerability reporting.
-
-For all other inquiries/communication, please refer to the [Reach Out to Us](https://github.com/ElementsProject/lightning?tab=readme-ov-file#reach-out-to-us) section in our README.
+To report security issues send an email to rusty@rustcorp.com.au, or
+security@blockstream.com (not for support).
 
 ## Signatures For Releases
 
 The following keys may be used to communicate sensitive information to
 developers, and to validate signatures on releases:
 
-| Name | Email | Fingerprint |
-|------|-------|-------------|
-| Blockstream Security Reporting | `security@blockstream.com` | 1176 542D A98E 71E1 3372  2EF7 4AC8 CC88 6844 A2D6 |
-| Rusty Russell | `rusty@rustcorp.com.au` | 15EE 8D6C AB0E 7F0C F999  BFCB D920 0E6C D1AD B8F1 |
-| Christian Decker | `decker@blockstream.com` | B731 AAC5 21B0 1385 9313  F674 A26D 6D9F E088 ED58 |
-| Lisa Neigut | `niftynei@gmail.com` | 30DE 693A E0DE 9E37 B3E7  EB6B BFF0 F678 10C1 EED1 |
-| Alex Myers | `alex@endothermic.dev` | 0437 4E42 789B BBA9 462E  4767 F3BF 63F2 7474 36AB |
-| Peter Neuroth | `pet.v.ne@gmail.com` | 653B 19F3 3DF7 EFF3 E9D1  C94C C3F2 1EE3 87FF 4CD2 |
-| Shahana Farooqui | `sfarooqui@blockstream.com` | FE13 58EB 7793 51DB 24E5  555A A327 573C 9758 9BF5 |
-| Blockstream CLN Release | `cln@blockstream.com` | 616C 52F9 9D06 12B2 A151  B107 4129 A994 AA7E 9852 |
+| Name | Fingerprint |
+|------|-------------|
+| Rusty Russell | 15EE 8D6C AB0E 7F0C F999  BFCB D920 0E6C D1AD B8F1 |
+| Christian Decker | B731 AAC5 21B0 1385 9313  F674 A26D 6D9F E088 ED58 |
+| Lisa Neigut | 30DE 693A E0DE 9E37 B3E7  EB6B BFF0 F678 10C1 EED1 |
+| Alex Myers | 0437 4E42 789B BBA9 462E  4767 F3BF 63F2 7474 36AB |
 
-You can import a key by running the following command with that individual’s fingerprint: 
-`gpg --keyserver hkps://keys.openpgp.org --recv-keys "<fingerprint>"`. 
-Ensure that you put quotes around fingerprints containing spaces.
+You can import a key by running the following command with that individual’s fingerprint: `gpg --keyserver hkps://keys.openpgp.org --recv-keys "<fingerprint>"` Ensure that you put quotes around fingerprints containing spaces.

action.yml

@@ -0,0 +1,6 @@
+---
+name: 'Lightning CI'
+description: 'A preconfigured container with all Core Lightning dependencies'
+runs:
+  using: 'docker'
+  image: 'contrib/Dockerfile.tester'

bitcoin/base58.c

@@ -47,14 +47,16 @@ static bool from_base58(u8 *version,
 			struct ripemd160 *rmd,
 			const char *base58, size_t base58_len)
 {
-	/* Initialize to avoid memcheck complaining if decoding a short value */
-	u8 buf[1 + sizeof(*rmd) + 4] = { 0 };
-	const size_t buflen = sizeof(buf);
-	const uint32_t flags = BASE58_FLAG_CHECKSUM;
+	u8 buf[1 + sizeof(*rmd) + 4];
+	/* Avoid memcheck complaining if decoding resulted in a short value */
+	size_t buflen = sizeof(buf);
+	memset(buf, 0, buflen);
+	char *terminated_base58 = tal_dup_arr(NULL, char, base58, base58_len, 1);
+	terminated_base58[base58_len] = '\0';
 
 	size_t written = 0;
-	int r = wally_base58_n_to_bytes(base58, base58_len, flags,
-					buf, buflen, &written);
+	int r = wally_base58_to_bytes(terminated_base58, BASE58_FLAG_CHECKSUM, buf, buflen, &written);
+	tal_free(terminated_base58);
 	if (r != WALLY_OK || written > buflen) {
 		return false;
 	}

bitcoin/block.c

@@ -5,7 +5,7 @@
 #include <bitcoin/tx.h>
 #include <ccan/mem/mem.h>
 #include <ccan/str/hex/hex.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 
 /* Sets *cursor to NULL and returns NULL when a pull fails. */
 static const u8 *pull(const u8 **cursor, size_t *max, void *copy, size_t n)
@@ -210,7 +210,7 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams,
 	b->tx = tal_arr(b, struct bitcoin_tx *, num);
 	b->txids = tal_arr(b, struct bitcoin_txid, num);
 	for (i = 0; i < num; i++) {
-		b->tx[i] = pull_bitcoin_tx_only(b->tx, &p, &len);
+		b->tx[i] = pull_bitcoin_tx(b->tx, &p, &len);
 		b->tx[i]->chainparams = chainparams;
 		bitcoin_txid(b->tx[i], &b->txids[i]);
 	}
@@ -237,14 +237,15 @@ static bool bitcoin_blkid_to_hex(const struct bitcoin_blkid *blockid,
 	return bitcoin_txid_to_hex(&fake_txid, hexstr, hexstr_len);
 }
 
-char *fmt_bitcoin_blkid(const tal_t *ctx,
-			const struct bitcoin_blkid *blkid)
+static char *fmt_bitcoin_blkid(const tal_t *ctx,
+			       const struct bitcoin_blkid *blkid)
 {
 	char *hexstr = tal_arr(ctx, char, hex_str_size(sizeof(*blkid)));
 
 	bitcoin_blkid_to_hex(blkid, hexstr, hex_str_size(sizeof(*blkid)));
 	return hexstr;
 }
+REGISTER_TYPE_TO_STRING(bitcoin_blkid, fmt_bitcoin_blkid);
 
 void fromwire_bitcoin_blkid(const u8 **cursor, size_t *max,
 			    struct bitcoin_blkid *blkid)

bitcoin/block.h

@@ -53,7 +53,4 @@ void fromwire_chainparams(const u8 **cursor, size_t *max,
 			  const struct chainparams **chainparams);
 void towire_chainparams(u8 **cursor, const struct chainparams *chainparams);
 
-char *fmt_bitcoin_blkid(const tal_t *ctx,
-			const struct bitcoin_blkid *blkid);
-
 #endif /* LIGHTNING_BITCOIN_BLOCK_H */

bitcoin/chainparams.c

@@ -137,32 +137,6 @@ const struct chainparams networks[] = {
      .bip32_key_version = {.bip32_pubkey_version = BIP32_VER_TEST_PUBLIC,
 			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
      .is_elements = false},
-    {.network_name = "testnet4",
-     .onchain_hrp = "tb",
-     .lightning_hrp = "tb",
-     .bip70_name = "testnet4",
-     // 00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043
-     .genesis_blockhash = {{{.u.u8 = {0x43, 0xf0, 0x8b, 0xda, 0xb0, 0x50, 0xe3,
-				      0x5b, 0x56, 0x7c, 0x86, 0x4b, 0x91, 0xf4,
-				      0x7f, 0x50, 0xae, 0x72, 0x5a, 0xe2, 0xde,
-				      0x53, 0xbc, 0xfb, 0xba, 0xf2, 0x84, 0xda,
-				      0x00, 0x00, 0x00, 0x00}}}},
-     .rpc_port = 48332,
-     .ln_port = 49735,
-     .cli = "bitcoin-cli",
-     .cli_args = "-testnet4",
-     .cli_min_supported_version = 150000,
-     .dust_limit = { 546 },
-     .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
-     .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
-     .p2pkh_version = 111,
-     .p2sh_version = 196,
-     .testnet = true,
-     .fee_asset_tag = NULL,
-     .bip32_key_version = {.bip32_pubkey_version = BIP32_VER_TEST_PUBLIC,
-			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
-     .is_elements = false},
     {.network_name = "litecoin",
      .onchain_hrp = "ltc",
      .lightning_hrp = "ltc",

bitcoin/chainparams.h

@@ -29,7 +29,7 @@ struct chainparams {
 	 *
 	 * - Bitcoin mainet with port number 9735 or the corresponding hexadecimal `0x2607`;
 	 * - Bitcoin testnet with port number 19735 (`0x4D17`);
-	 * - Bitcoin signet with port number 39735 (`0x9B37`).
+	 * - Bitcoin signet with port number 39735 (`0xF87`).
 	 */
 	const int ln_port;
 	const char *cli;

bitcoin/locktime.c

@@ -2,6 +2,7 @@
 #include <assert.h>
 #include <bitcoin/locktime.h>
 #include <ccan/tal/str/str.h>
+#include <common/type_to_string.h>
 
 #define SECONDS_POINT 500000000
 

bitcoin/preimage.c

@@ -1,6 +1,5 @@
 #include "config.h"
 #include <bitcoin/preimage.h>
-#include <common/utils.h>
 #include <wire/wire.h>
 
 void fromwire_preimage(const u8 **cursor, size_t *max, struct preimage *preimage)
@@ -13,7 +12,4 @@ void towire_preimage(u8 **pptr, const struct preimage *preimage)
 	towire(pptr, preimage, sizeof(*preimage));
 }
 
-char *fmt_preimage(const tal_t *ctx, const struct preimage *preimage)
-{
-	return tal_hexstr(ctx, preimage, sizeof(*preimage));
-}
+

bitcoin/preimage.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/short_types/short_types.h>
 #include <ccan/structeq/structeq.h>
-#include <ccan/tal/tal.h>
 
 struct preimage {
 	u8 r[32];
@@ -14,5 +13,4 @@ STRUCTEQ_DEF(preimage, 0, r);
 void fromwire_preimage(const u8 **cursor, size_t *max, struct preimage *preimage);
 void towire_preimage(u8 **pptr, const struct preimage *preimage);
 
-char *fmt_preimage(const tal_t *ctx, const struct preimage *preimage);
 #endif /* LIGHTNING_BITCOIN_PREIMAGE_H */

bitcoin/privkey.c

@@ -1,10 +1,10 @@
 #include "config.h"
 #include <bitcoin/privkey.h>
 #include <ccan/str/hex/hex.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
-char *fmt_privkey(const tal_t *ctx, const struct privkey *secret)
+static char *privkey_to_hexstr(const tal_t *ctx, const struct privkey *secret)
 {
 	/* Bitcoin appends "01" to indicate the pubkey is compressed. */
 	char *str = tal_arr(ctx, char, hex_str_size(sizeof(*secret) + 1));
@@ -12,11 +12,8 @@ char *fmt_privkey(const tal_t *ctx, const struct privkey *secret)
 	strcat(str, "01");
 	return str;
 }
-
-char *fmt_secret(const tal_t *ctx, const struct secret *secret)
-{
-	return tal_hexstr(ctx, secret, sizeof(*secret));
-}
+REGISTER_TYPE_TO_STRING(privkey, privkey_to_hexstr);
+REGISTER_TYPE_TO_HEXSTR(secret);
 
 bool secret_eq_consttime(const struct secret *a, const struct secret *b)
 {

bitcoin/privkey.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/short_types/short_types.h>
 #include <ccan/structeq/structeq.h>
-#include <ccan/tal/tal.h>
 
 #define PRIVKEY_LEN 32
 
@@ -26,7 +25,4 @@ void fromwire_privkey(const u8 **cursor, size_t *max, struct privkey *privkey);
 void towire_privkey(u8 **pptr, const struct privkey *privkey);
 void towire_secret(u8 **pptr, const struct secret *secret);
 
-char *fmt_privkey(const tal_t *ctx, const struct privkey *privkey);
-char *fmt_secret(const tal_t *ctx, const struct secret *secret);
-
 #endif /* LIGHTNING_BITCOIN_PRIVKEY_H */

bitcoin/psbt.c

@@ -8,7 +8,7 @@
 #include <ccan/ccan/array_size/array_size.h>
 #include <ccan/ccan/mem/mem.h>
 #include <ccan/tal/str/str.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wally_psbt.h>
 #include <wire/wire.h>
 
@@ -36,7 +36,7 @@ struct wally_psbt *create_psbt(const tal_t *ctx, size_t num_inputs, size_t num_o
 	return psbt;
 }
 
-struct wally_psbt *clone_psbt(const tal_t *ctx, const struct wally_psbt *psbt)
+struct wally_psbt *clone_psbt(const tal_t *ctx, struct wally_psbt *psbt)
 {
 	struct wally_psbt *clone;
 	tal_wally_start();
@@ -86,95 +86,6 @@ struct wally_psbt *new_psbt(const tal_t *ctx, const struct wally_tx *wtx)
 	return psbt;
 }
 
-struct wally_psbt *combine_psbt(const tal_t *ctx,
-				const struct wally_psbt *psbt0,
-				const struct wally_psbt *psbt1)
-{
-	struct wally_psbt *combined_psbt;
-	tal_wally_start();
-	if (wally_psbt_clone_alloc(psbt0, 0, &combined_psbt) != WALLY_OK)
-		abort();
-	if (wally_psbt_combine(combined_psbt, psbt1) != WALLY_OK) {
-		tal_wally_end_onto(ctx, combined_psbt, struct wally_psbt);
-		return tal_free(combined_psbt);
-	}
-	tal_wally_end_onto(ctx, combined_psbt, struct wally_psbt);
-	return combined_psbt;
-}
-
-static bool parent_or_grandparent(const tal_t *goal, const tal_t *child)
-{
-	const tal_t *parent = tal_parent(child);
-	if (!parent)
-		return false;
-	return parent == goal || parent_or_grandparent(goal, parent);
-}
-
-#define NULL_OR_MATCH_P(item, parent) \
-	((item) == NULL || parent_or_grandparent((parent), (item)))
-
-static void audit_map(const tal_t *ctx, const struct wally_map *map)
-{
-	assert(NULL_OR_MATCH_P(map->items, ctx));
-	for (size_t i = 0; i < map->num_items; i++) {
-		assert(NULL_OR_MATCH_P(map->items[i].key, ctx));
-		assert(NULL_OR_MATCH_P(map->items[i].value, ctx));
-		assert(!map->items[i].key
-			|| tal_bytelen(map->items[i].key)
-				== map->items[i].key_len);
-		assert(!map->items[i].value
-			|| tal_bytelen(map->items[i].value)
-				== map->items[i].value_len);
-	}
-}
-
-void audit_psbt(const tal_t *ctx, const struct wally_psbt *psbt)
-{
-	assert(psbt);
-	assert(NULL_OR_MATCH_P(psbt->tx, ctx));
-	assert(NULL_OR_MATCH_P(psbt->inputs, ctx));
-	assert(NULL_OR_MATCH_P(psbt->outputs, ctx));
-	audit_map(ctx, &psbt->unknowns);
-	audit_map(ctx, &psbt->global_xpubs);
-#ifndef WALLY_ABI_NO_ELEMENTS
-	audit_map(ctx, &psbt->global_scalars);
-#endif
-	for (size_t i = 0; i < psbt->num_inputs; i++) {
-		assert(NULL_OR_MATCH_P(psbt->inputs[i].utxo, ctx));
-		assert(NULL_OR_MATCH_P(psbt->inputs[i].witness_utxo, ctx));
-		assert(NULL_OR_MATCH_P(psbt->inputs[i].final_witness, ctx));
-		audit_map(ctx, &psbt->inputs[i].keypaths);
-		audit_map(ctx, &psbt->inputs[i].signatures);
-		audit_map(ctx, &psbt->inputs[i].unknowns);
-		audit_map(ctx, &psbt->inputs[i].preimages);
-		audit_map(ctx, &psbt->inputs[i].psbt_fields);
-		audit_map(ctx, &psbt->inputs[i].taproot_leaf_signatures);
-		audit_map(ctx, &psbt->inputs[i].taproot_leaf_scripts);
-		audit_map(ctx, &psbt->inputs[i].taproot_leaf_hashes);
-		audit_map(ctx, &psbt->inputs[i].taproot_leaf_paths);
-		/* DTODO: Investigate if taproot wally maps have child maps */
-#ifndef WALLY_ABI_NO_ELEMENTS
-		assert(NULL_OR_MATCH_P(psbt->inputs[i].pegin_tx, ctx));
-		assert(NULL_OR_MATCH_P(psbt->inputs[i].pegin_witness, ctx));
-		audit_map(ctx, &psbt->inputs[i].pset_fields);
-#endif
-	}
-	for (size_t i = 0; i < psbt->num_outputs; i++) {
-		assert(NULL_OR_MATCH_P(psbt->outputs[i].script, ctx));
-		assert(psbt->outputs[i].script_len
-			== tal_bytelen(psbt->outputs[i].script));
-		audit_map(ctx, &psbt->outputs[i].keypaths);
-		audit_map(ctx, &psbt->outputs[i].unknowns);
-		audit_map(ctx, &psbt->outputs[i].psbt_fields);
-		audit_map(ctx, &psbt->outputs[i].taproot_tree);
-		audit_map(ctx, &psbt->outputs[i].taproot_leaf_hashes);
-		audit_map(ctx, &psbt->outputs[i].taproot_leaf_paths);
-#ifndef WALLY_ABI_NO_ELEMENTS
-		audit_map(ctx, &psbt->outputs[i].pset_fields);
-#endif
-	}
-}
-
 bool psbt_is_finalized(const struct wally_psbt *psbt)
 {
 	size_t is_finalized;
@@ -363,57 +274,6 @@ bool psbt_input_set_signature(struct wally_psbt *psbt, size_t in,
 	return ok;
 }
 
-bool psbt_input_have_signature(const struct wally_psbt *psbt,
-			       size_t in,
-			       const struct pubkey *pubkey,
-			       bool *signature_found)
-{
-	u8 pk_der[PUBKEY_CMPR_LEN];
-	size_t index_plus_one;
-	bool ok;
-
-	assert(in < psbt->num_inputs);
-
-	pubkey_to_der(pk_der, pubkey);
-
-	ok = wally_psbt_input_find_signature(&psbt->inputs[in], pk_der,
-					     sizeof(pk_der),
-					     &index_plus_one) == WALLY_OK;
-	if (ok)
-		*signature_found = index_plus_one > 0;
-	return ok;
-}
-
-bool psbt_input_get_ecdsa_sig(const tal_t *ctx,
-			      const struct wally_psbt *psbt,
-			      size_t in,
-			      const struct pubkey *pubkey,
-			      struct bitcoin_signature **sig)
-{
-	u8 pk_der[PUBKEY_CMPR_LEN];
-	size_t index_plus_one;
-	struct wally_map_item *item;
-	bool ok;
-
-	assert(in < psbt->num_inputs);
-
-	pubkey_to_der(pk_der, pubkey);
-	*sig = NULL;
-
-	ok = wally_psbt_input_find_signature(&psbt->inputs[in], pk_der,
-					     sizeof(pk_der),
-					     &index_plus_one) == WALLY_OK;
-	if (ok) {
-		item = &psbt->inputs[in].signatures.items[index_plus_one - 1];
-		*sig = tal(ctx, struct bitcoin_signature);
-		if (!signature_from_der(item->value, item->value_len, *sig)) {
-			*sig = tal_free(*sig);
-			return false;
-		}
-	}
-	return ok;
-}
-
 void psbt_input_set_wit_utxo(struct wally_psbt *psbt, size_t in,
 			     const u8 *scriptPubkey, struct amount_sat amt)
 {
@@ -552,9 +412,10 @@ void psbt_elements_normalize_fees(struct wally_psbt *psbt)
 }
 
 void wally_psbt_input_get_txid(const struct wally_psbt_input *in,
-			       struct bitcoin_txid *txid)
+                 struct bitcoin_txid *txid)
 {
-	CROSS_TYPE_ASSIGNMENT(txid, &in->txhash);
+    BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+    memcpy(txid, in->txhash, sizeof(struct bitcoin_txid));
 }
 
 bool psbt_has_input(const struct wally_psbt *psbt,
@@ -677,18 +538,37 @@ u8 *psbt_make_key(const tal_t *ctx, u8 key_subtype, const u8 *key_data)
 	return key;
 }
 
-static void map_replace(const tal_t *ctx,
-			struct wally_map *map,
-			const u8 *key,
-			const void *value,
-			size_t value_len)
+static bool wally_map_set_unknown(const tal_t *ctx,
+				  struct wally_map *map,
+				  const u8 *key,
+				  const void *value,
+				  size_t value_len)
 {
-	const unsigned char *checked_value = memcheck(value, value_len);
-	tal_wally_start();
-	if (wally_map_replace(map, key, tal_bytelen(key),
-			      checked_value, value_len) != WALLY_OK)
-		abort();
-	tal_wally_end(ctx);
+	size_t exists_at;
+	struct wally_map_item *item;
+
+	assert(value_len != 0);
+	if (wally_map_find(map, key, tal_bytelen(key), &exists_at) != WALLY_OK)
+		return false;
+
+	/* If not exists, add */
+	if (exists_at == 0) {
+		bool ok;
+		tal_wally_start();
+		ok = wally_map_add(map, key, tal_bytelen(key),
+			      (unsigned char *) memcheck(value, value_len), value_len)
+			== WALLY_OK;
+		tal_wally_end(ctx);
+		return ok;
+	}
+
+	/* Already in map, update entry */
+	item = &map->items[exists_at - 1];
+	tal_resize(&item->value, value_len);
+	memcpy(item->value, memcheck(value, value_len), value_len);
+	item->value_len = value_len;
+
+	return true;
 }
 
 void psbt_input_set_unknown(const tal_t *ctx,
@@ -697,33 +577,39 @@ void psbt_input_set_unknown(const tal_t *ctx,
 			    const void *value,
 			    size_t value_len)
 {
-	map_replace(ctx, &in->unknowns, key, value, value_len);
+	if (!wally_map_set_unknown(ctx, &in->unknowns, key, value, value_len))
+		abort();
+}
+
+static void *psbt_get_unknown(const struct wally_map *map,
+			      const u8 *key,
+			      size_t *val_len)
+{
+	size_t index;
+
+	if (wally_map_find(map, key, tal_bytelen(key), &index) != WALLY_OK)
+		return NULL;
+
+	/* Zero: item not found. */
+	if (index == 0)
+		return NULL;
+
+	/* ++: item is at this index minus 1 */
+	*val_len = map->items[index - 1].value_len;
+	return map->items[index - 1].value;
 }
 
 void *psbt_get_lightning(const struct wally_map *map,
 			 const u8 proprietary_type,
 			 size_t *val_len)
 {
+	void *res;
 	u8 *key = psbt_make_key(NULL, proprietary_type, NULL);
-	const struct wally_map_item *item;
-	item = wally_map_get(map, key, tal_bytelen(key));
+	res = psbt_get_unknown(map, key, val_len);
 	tal_free(key);
-	if (!item)
-		return NULL;
-	*val_len = item->value_len;
-	return item->value;
+	return res;
 }
 
-void psbt_set_lightning(const tal_t *ctx,
-			struct wally_map *map,
-			const u8 proprietary_type,
-			const void *value,
-			size_t val_len)
-{
-	u8 *key = psbt_make_key(NULL, proprietary_type, NULL);
-	map_replace(ctx, map, key, value, val_len);
-	tal_free(key);
-}
 
 void psbt_output_set_unknown(const tal_t *ctx,
 			     struct wally_psbt_output *out,
@@ -731,7 +617,8 @@ void psbt_output_set_unknown(const tal_t *ctx,
 			     const void *value,
 			     size_t value_len)
 {
-	map_replace(ctx, &out->unknowns, key, value, value_len);
+	if (!wally_map_set_unknown(ctx, &out->unknowns, key, value, value_len))
+		abort();
 }
 
 /* Use the destructor to free the wally_tx */
@@ -834,9 +721,10 @@ struct wally_psbt *psbt_from_b64(const tal_t *ctx,
 				 size_t b64len)
 {
 	struct wally_psbt *psbt;
+	char *str = tal_strndup(tmpctx, b64, b64len);
 
 	tal_wally_start();
-	if (wally_psbt_from_base64_n(b64, b64len, /* flags */ 0, &psbt) == WALLY_OK)
+	if (wally_psbt_from_base64(str, /* flags */ 0, &psbt) == WALLY_OK)
 		tal_add_destructor(psbt, psbt_destroy);
 	else
 		psbt = NULL;
@@ -845,7 +733,7 @@ struct wally_psbt *psbt_from_b64(const tal_t *ctx,
 	return psbt;
 }
 
-char *fmt_wally_psbt(const tal_t *ctx, const struct wally_psbt *psbt)
+char *psbt_to_b64(const tal_t *ctx, const struct wally_psbt *psbt)
 {
 	char *serialized_psbt;
 	int ret;
@@ -857,6 +745,7 @@ char *fmt_wally_psbt(const tal_t *ctx, const struct wally_psbt *psbt)
 
 	return serialized_psbt;
 }
+REGISTER_TYPE_TO_STRING(wally_psbt, psbt_to_b64);
 
 const u8 *psbt_get_bytes(const tal_t *ctx, const struct wally_psbt *psbt,
 			 size_t *bytes_written)
@@ -941,6 +830,18 @@ struct wally_psbt *fromwire_wally_psbt(const tal_t *ctx,
 	if (!psbt)
 		return fromwire_fail(cursor, max);
 
+#if DEVELOPER
+	/* Re-marshall for sanity check! */
+	u8 *tmpbuf = tal_arr(NULL, u8, psbt_byte_len);
+	size_t written;
+	if (wally_psbt_to_bytes(psbt, 0, tmpbuf, psbt_byte_len, &written) != WALLY_OK) {
+		tal_free(tmpbuf);
+		tal_free(psbt);
+		return fromwire_fail(cursor, max);
+	}
+	tal_free(tmpbuf);
+#endif
+
 	/* Internally we always operate on v2 */
 	psbt_set_version(psbt, 2);
 
@@ -998,21 +899,25 @@ struct amount_sat psbt_compute_fee(const struct wally_psbt *psbt)
 }
 
 bool wally_psbt_input_spends(const struct wally_psbt_input *input,
-			     const struct bitcoin_outpoint *outpoint)
+               const struct bitcoin_outpoint *outpoint)
 {
-	/* Useful, as tx_part can have some NULL inputs */
-	if (!input)
-		return false;
+    /* Useful, as tx_part can have some NULL inputs */
+    if (!input)
+        return false;
+    BUILD_ASSERT(sizeof(outpoint->txid) == sizeof(input->txhash));
 	if (input->index != outpoint->n)
 		return false;
-	return CROSS_TYPE_EQ(&outpoint->txid, &input->txhash);
+    if (memcmp(&outpoint->txid, input->txhash, sizeof(outpoint->txid)) != 0)
+        return false;
+    return true;
 }
 
 void wally_psbt_input_get_outpoint(const struct wally_psbt_input *in,
-				   struct bitcoin_outpoint *outpoint)
+                 struct bitcoin_outpoint *outpoint)
 {
-	CROSS_TYPE_ASSIGNMENT(&outpoint->txid, &in->txhash);
-	outpoint->n = in->index;
+    BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+    memcpy(&outpoint->txid, in->txhash, sizeof(struct bitcoin_txid));
+    outpoint->n = in->index;
 }
 
 const u8 *wally_psbt_output_get_script(const tal_t *ctx,

bitcoin/psbt.h

@@ -48,29 +48,7 @@ struct wally_psbt *new_psbt(const tal_t *ctx,
  * @ctx - allocation context
  * @psbt - psbt to be cloned
  */
-struct wally_psbt *clone_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
-
-/**
- * combine_psbt - Combine two PSBT into a cloned copy
- *
- * @ctx - allocation context
- * @psbt0 - one psbt
- * @psbt1 - other psbt
- */
-struct wally_psbt *combine_psbt(const tal_t *ctx,
-				const struct wally_psbt *psbt0,
-				const struct wally_psbt *psbt1);
-
-/**
- * audit_psbt - Audit the memory structure of the PSBT.
- *
- * This checks all known memory allocations in the PSBT and asserts that they
- * are all allocated with 'ctx' being it's parent.
- *
- * ctx - the ctx all memory *should* be attached to
- * psbt - the PSBT to audit.
- * */
-void audit_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
+struct wally_psbt *clone_psbt(const tal_t *ctx, struct wally_psbt *psbt);
 
 /**
  * psbt_is_finalized - Check if tx is ready to be extracted
@@ -189,22 +167,6 @@ WARN_UNUSED_RESULT bool psbt_input_set_signature(struct wally_psbt *psbt, size_t
 						 const struct pubkey *pubkey,
 						 const struct bitcoin_signature *sig);
 
-/* Returns false on error. On success, *signature_found is set to true if the
- * input has a signature present for `pubkey` and false if if one was not found.
- * Only signature presence is checked, it is not validated. */
-WARN_UNUSED_RESULT bool psbt_input_have_signature(const struct wally_psbt *psbt,
-						  size_t in,
-						  const struct pubkey *pubkey,
-						  bool *signature_found);
-
-/* Returns false on error. On success *sig is set to the signature otherwise
- * *sig is set to NULL. */
-WARN_UNUSED_RESULT bool psbt_input_get_ecdsa_sig(const tal_t *ctx,
-						 const struct wally_psbt *psbt,
-						 size_t in,
-						 const struct pubkey *pubkey,
-						 struct bitcoin_signature **sig);
-
 void psbt_input_set_witscript(struct wally_psbt *psbt, size_t in, const u8 *wscript);
 
 /* psbt_input_set_unknown - Set the given Key-Value in the psbt's input keymap
@@ -231,19 +193,6 @@ void *psbt_get_lightning(const struct wally_map *map,
 			 const u8 proprietary_type,
 			 size_t *val_len);
 
-/* psbt_set_lightning - Set a propreitary lightning value on the given map
- *
- * @map - map of unknowns to set the value
- * @proprietary_type - type no. to set
- * @value - the value to be set
- * @val_len - length of value
- */
-void psbt_set_lightning(const tal_t *ctx,
-			struct wally_map *map,
-			const u8 proprietary_type,
-			const void *value,
-			size_t val_len);
-
 /* psbt_output_set_unknown - Set the given Key-Value in the psbt's output keymap
  *
  * @ctx - tal context for allocations
@@ -325,7 +274,7 @@ bool elements_psbt_output_is_fee(const struct wally_psbt *psbt, size_t outnum);
 struct wally_psbt *psbt_from_b64(const tal_t *ctx,
 				 const char *b64,
 				 size_t b64len);
-char *fmt_wally_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
+char *psbt_to_b64(const tal_t *ctx, const struct wally_psbt *psbt);
 const u8 *psbt_get_bytes(const tal_t *ctx, const struct wally_psbt *psbt,
 			 size_t *bytes_written);
 bool validate_psbt(const struct wally_psbt *psbt);

bitcoin/pubkey.c

@@ -4,7 +4,7 @@
 #include <bitcoin/pubkey.h>
 #include <ccan/mem/mem.h>
 #include <ccan/str/hex/hex.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
 #ifndef SUPERVERBOSE
@@ -62,15 +62,16 @@ bool pubkey_from_hexstr(const char *derstr, size_t slen, struct pubkey *key)
 	return pubkey_from_der(der, dlen, key);
 }
 
-char *fmt_pubkey(const tal_t *ctx, const struct pubkey *key)
+char *pubkey_to_hexstr(const tal_t *ctx, const struct pubkey *key)
 {
 	unsigned char der[PUBKEY_CMPR_LEN];
 
 	pubkey_to_der(der, key);
 	return tal_hexstr(ctx, der, sizeof(der));
 }
+REGISTER_TYPE_TO_STRING(pubkey, pubkey_to_hexstr);
 
-char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key)
+static char *secp256k1_pubkey_to_hexstr(const tal_t *ctx, const secp256k1_pubkey *key)
 {
 	unsigned char der[PUBKEY_CMPR_LEN];
 	size_t outlen = sizeof(der);
@@ -80,10 +81,11 @@ char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key)
 	assert(outlen == sizeof(der));
 	return tal_hexstr(ctx, der, sizeof(der));
 }
+REGISTER_TYPE_TO_STRING(secp256k1_pubkey, secp256k1_pubkey_to_hexstr);
 
 int pubkey_cmp(const struct pubkey *a, const struct pubkey *b)
 {
-	u8 keya[PUBKEY_CMPR_LEN], keyb[PUBKEY_CMPR_LEN];
+	u8 keya[33], keyb[33];
 	pubkey_to_der(keya, a);
 	pubkey_to_der(keyb, b);
 	return memcmp(keya, keyb, sizeof(keya));

bitcoin/pubkey.h

@@ -23,8 +23,7 @@ STRUCTEQ_DEF(pubkey, 0, pubkey.data);
 bool pubkey_from_hexstr(const char *derstr, size_t derlen, struct pubkey *key);
 
 /* Convert from hex string of DER (scriptPubKey from validateaddress) */
-char *fmt_pubkey(const tal_t *ctx, const struct pubkey *key);
-char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key);
+char *pubkey_to_hexstr(const tal_t *ctx, const struct pubkey *key);
 
 /* Point from secret */
 bool pubkey_from_secret(const struct secret *secret, struct pubkey *key);

bitcoin/script.c

@@ -476,8 +476,10 @@ u8 *p2wpkh_scriptcode(const tal_t *ctx, const struct pubkey *key)
 	return script;
 }
 
-bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
+bool is_p2pkh(const u8 *script, struct bitcoin_address *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2PKH_LEN)
 		return false;
 	if (script[0] != OP_DUP)
@@ -495,8 +497,10 @@ bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
 	return true;
 }
 
-bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr)
+bool is_p2sh(const u8 *script, struct ripemd160 *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2SH_LEN)
 		return false;
 	if (script[0] != OP_HASH160)
@@ -510,8 +514,10 @@ bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr)
 	return true;
 }
 
-bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr)
+bool is_p2wsh(const u8 *script, struct sha256 *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2WSH_LEN)
 		return false;
 	if (script[0] != OP_0)
@@ -523,8 +529,10 @@ bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr)
 	return true;
 }
 
-bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
+bool is_p2wpkh(const u8 *script, struct bitcoin_address *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN)
 		return false;
 	if (script[0] != OP_0)
@@ -536,8 +544,10 @@ bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr
 	return true;
 }
 
-bool is_p2tr(const u8 *script, size_t script_len, u8 xonly_pubkey[32])
+bool is_p2tr(const u8 *script, u8 xonly_pubkey[32])
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2TR_LEN)
 		return false;
 	if (script[0] != OP_1)
@@ -550,20 +560,17 @@ bool is_p2tr(const u8 *script, size_t script_len, u8 xonly_pubkey[32])
 	return true;
 }
 
-bool is_known_scripttype(const u8 *script, size_t script_len)
+bool is_known_scripttype(const u8 *script)
 {
-	return is_p2wpkh(script, script_len, NULL)
-		|| is_p2wsh(script, script_len, NULL)
-		|| is_p2sh(script, script_len, NULL)
-		|| is_p2pkh(script, script_len, NULL)
-		|| is_p2tr(script, script_len, NULL);
+	return is_p2wpkh(script, NULL) || is_p2wsh(script, NULL)
+		|| is_p2sh(script, NULL) || is_p2pkh(script, NULL)
+		|| is_p2tr(script, NULL);
 }
 
-bool is_known_segwit_scripttype(const u8 *script, size_t script_len)
+bool is_known_segwit_scripttype(const u8 *script)
 {
-	return is_p2wpkh(script, script_len, NULL)
-		|| is_p2wsh(script, script_len, NULL)
-		|| is_p2tr(script, script_len, NULL);
+	return is_p2wpkh(script, NULL) || is_p2wsh(script, NULL)
+		|| is_p2tr(script, NULL);
 }
 
 u8 **bitcoin_witness_sig_and_element(const tal_t *ctx,
@@ -986,8 +993,12 @@ bool is_anchor_witness_script(const u8 *script, size_t script_len)
 
 bool scripteq(const u8 *s1, const u8 *s2)
 {
-	size_t s1_len = tal_count(s1), s2_len = tal_count(s2);
-	memcheck(s1, s1_len);
-	memcheck(s2, s2_len);
-	return memeq(s1, s1_len, s2, s2_len);
+	memcheck(s1, tal_count(s1));
+	memcheck(s2, tal_count(s2));
+
+	if (tal_count(s1) != tal_count(s2))
+		return false;
+	if (tal_count(s1) == 0)
+		return true;
+	return memcmp(s1, s2, tal_count(s1)) == 0;
 }

bitcoin/script.h

@@ -159,25 +159,25 @@ u8 *bitcoin_wscript_anchor(const tal_t *ctx,
 			   const struct pubkey *funding_pubkey);
 
 /* Is this a pay to pubkey hash? (extract addr if not NULL) */
-bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr);
+bool is_p2pkh(const u8 *script, struct bitcoin_address *addr);
 
 /* Is this a pay to script hash? (extract addr if not NULL) */
-bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr);
+bool is_p2sh(const u8 *script, struct ripemd160 *addr);
 
 /* Is this (version 0) pay to witness script hash? (extract addr if not NULL) */
-bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr);
+bool is_p2wsh(const u8 *script, struct sha256 *addr);
 
 /* Is this (version 0) pay to witness pubkey hash? (extract addr if not NULL) */
-bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr);
+bool is_p2wpkh(const u8 *script, struct bitcoin_address *addr);
 
 /* Is this a taproot output? (extract xonly_pubkey bytes if not NULL) */
-bool is_p2tr(const u8 *script, size_t script_len, u8 xonly_pubkey[32]);
+bool is_p2tr(const u8 *script, u8 xonly_pubkey[32]);
 
 /* Is this one of the above script types? */
-bool is_known_scripttype(const u8 *script, size_t script_len);
+bool is_known_scripttype(const u8 *script);
 
 /* Is this a witness script type? */
-bool is_known_segwit_scripttype(const u8 *script, size_t script_len);
+bool is_known_segwit_scripttype(const u8 *script);
 
 /* Is this a to-remote witness script (used for option_anchor_outputs)? */
 bool is_to_remote_anchored_witness_script(const u8 *script, size_t script_len);

bitcoin/shadouble.c

@@ -1,7 +1,7 @@
 #include "config.h"
 #include <bitcoin/shadouble.h>
 #include <ccan/mem/mem.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
 void sha256_double(struct sha256_double *shadouble, const void *p, size_t len)
@@ -15,11 +15,7 @@ void sha256_double_done(struct sha256_ctx *shactx, struct sha256_double *res)
 	sha256_done(shactx, &res->sha);
 	sha256(&res->sha, &res->sha, sizeof(res->sha));
 }
-
-char *fmt_sha256_double(const tal_t *ctx, const struct sha256_double *shad)
-{
-	return tal_hexstr(ctx, shad, sizeof(*shad));
-}
+REGISTER_TYPE_TO_HEXSTR(sha256_double);
 
 void towire_sha256_double(u8 **pptr, const struct sha256_double *sha256d)
 {

bitcoin/shadouble.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/crypto/sha256/sha256.h>
 #include <ccan/short_types/short_types.h>
-#include <ccan/tal/tal.h>
 
 /* To explicitly distinguish between single sha and bitcoin's standard double */
 struct sha256_double {
@@ -18,6 +17,4 @@ void sha256_double_done(struct sha256_ctx *sha256, struct sha256_double *res);
 void fromwire_sha256_double(const u8 **cursor, size_t *max,
 			    struct sha256_double *sha256d);
 void towire_sha256_double(u8 **pptr, const struct sha256_double *sha256d);
-
-char *fmt_sha256_double(const tal_t *ctx, const struct sha256_double *shad);
 #endif /* LIGHTNING_BITCOIN_SHADOUBLE_H */

bitcoin/short_channel_id.c

@@ -1,6 +1,7 @@
 #include "config.h"
 #include <bitcoin/short_channel_id.h>
 #include <ccan/tal/str/str.h>
+#include <common/type_to_string.h>
 #include <stdio.h>
 #include <wire/wire.h>
 
@@ -52,7 +53,7 @@ bool short_channel_id_from_str(const char *str, size_t strlen,
 		&& mk_short_channel_id(dst, blocknum, txnum, outnum);
 }
 
-char *fmt_short_channel_id(const tal_t *ctx, struct short_channel_id scid)
+char *short_channel_id_to_str(const tal_t *ctx, const struct short_channel_id *scid)
 {
 	return tal_fmt(ctx, "%dx%dx%d",
 		       short_channel_id_blocknum(scid),
@@ -77,25 +78,26 @@ bool short_channel_id_dir_from_str(const char *str, size_t strlen,
 	return true;
 }
 
-char *fmt_short_channel_id_dir(const tal_t *ctx,
-			       const struct short_channel_id_dir *scidd)
+static char *short_channel_id_dir_to_str(const tal_t *ctx,
+					 const struct short_channel_id_dir *scidd)
 {
-	char *str, *scidstr = fmt_short_channel_id(NULL, scidd->scid);
+	char *str, *scidstr = short_channel_id_to_str(NULL, &scidd->scid);
 	str = tal_fmt(ctx, "%s/%u", scidstr, scidd->dir);
 	tal_free(scidstr);
 	return str;
 }
 
+REGISTER_TYPE_TO_STRING(short_channel_id, short_channel_id_to_str);
+REGISTER_TYPE_TO_STRING(short_channel_id_dir, short_channel_id_dir_to_str);
+
 void towire_short_channel_id(u8 **pptr,
-			     struct short_channel_id short_channel_id)
+			     const struct short_channel_id *short_channel_id)
 {
-	towire_u64(pptr, short_channel_id.u64);
+	towire_u64(pptr, short_channel_id->u64);
 }
 
-struct short_channel_id fromwire_short_channel_id(const u8 **cursor, size_t *max)
+void fromwire_short_channel_id(const u8 **cursor, size_t *max,
+			       struct short_channel_id *short_channel_id)
 {
-	struct short_channel_id scid;
-
-	scid.u64 = fromwire_u64(cursor, max);
-	return scid;
+	short_channel_id->u64 = fromwire_u64(cursor, max);
 }

bitcoin/short_channel_id.h

@@ -3,6 +3,7 @@
 #include "config.h"
 #include <ccan/compiler/compiler.h>
 #include <ccan/short_types/short_types.h>
+#include <ccan/structeq/structeq.h>
 #include <ccan/tal/tal.h>
 #include <common/gossip_constants.h>
 
@@ -11,18 +12,8 @@
 struct short_channel_id {
 	u64 u64;
 };
-
-static inline bool short_channel_id_eq(struct short_channel_id a,
-				       struct short_channel_id b)
-{
-	return a.u64 == b.u64;
-}
-
-static inline size_t short_channel_id_hash(struct short_channel_id scid)
-{
-	/* scids cost money to generate, so simple hash works here */
-	return (scid.u64 >> 32) ^ (scid.u64 >> 16) ^ scid.u64;
-}
+/* Define short_channel_id_eq (no padding) */
+STRUCTEQ_DEF(short_channel_id, 0, u64);
 
 /* BOLT #7:
  *
@@ -41,38 +32,32 @@ struct short_channel_id_dir {
 	int dir;
 };
 
-static inline bool short_channel_id_dir_eq(const struct short_channel_id_dir *a,
-					   const struct short_channel_id_dir *b)
+static inline u32 short_channel_id_blocknum(const struct short_channel_id *scid)
 {
-	return short_channel_id_eq(a->scid, b->scid) && a->dir == b->dir;
+	return scid->u64 >> 40;
 }
 
-static inline u32 short_channel_id_blocknum(struct short_channel_id scid)
+static inline bool is_stub_scid(const struct short_channel_id *scid)
 {
-	return scid.u64 >> 40;
+	return scid ? scid->u64 >> 40 == 1 &&
+		((scid->u64 >> 16) & 0x00FFFFFF) == 1 &&
+		(scid->u64 & 0xFFFF) == 1 : false;
 }
 
-static inline bool is_stub_scid(struct short_channel_id scid)
+static inline u32 short_channel_id_txnum(const struct short_channel_id *scid)
 {
-	return scid.u64 >> 40 == 1 &&
-		((scid.u64 >> 16) & 0x00FFFFFF) == 1 &&
-		(scid.u64 & 0xFFFF) == 1;
+	return (scid->u64 >> 16) & 0x00FFFFFF;
 }
 
-static inline u32 short_channel_id_txnum(struct short_channel_id scid)
+static inline u16 short_channel_id_outnum(const struct short_channel_id *scid)
 {
-	return (scid.u64 >> 16) & 0x00FFFFFF;
-}
-
-static inline u16 short_channel_id_outnum(struct short_channel_id scid)
-{
-	return scid.u64 & 0xFFFF;
+	return scid->u64 & 0xFFFF;
 }
 
 /* Subtly, at block N, depth is 1, hence the -1 here. eg. 103x1x0 is announceable
  * when height is 108. */
 static inline bool
-is_scid_depth_announceable(struct short_channel_id scid,
+is_scid_depth_announceable(const struct short_channel_id *scid,
 			  unsigned int height)
 {
 	return short_channel_id_blocknum(scid) + ANNOUNCE_MIN_DEPTH - 1
@@ -86,16 +71,15 @@ bool WARN_UNUSED_RESULT mk_short_channel_id(struct short_channel_id *scid,
 bool WARN_UNUSED_RESULT short_channel_id_from_str(const char *str, size_t strlen,
 						  struct short_channel_id *dst);
 
+char *short_channel_id_to_str(const tal_t *ctx, const struct short_channel_id *scid);
+
 bool WARN_UNUSED_RESULT short_channel_id_dir_from_str(const char *str, size_t strlen,
 						      struct short_channel_id_dir *scidd);
 
-char *fmt_short_channel_id(const tal_t *ctx, struct short_channel_id scid);
-char *fmt_short_channel_id_dir(const tal_t *ctx,
-			       const struct short_channel_id_dir *scidd);
-
 /* Marshal/unmarshal */
 void towire_short_channel_id(u8 **pptr,
-			     struct short_channel_id short_channel_id);
-struct short_channel_id fromwire_short_channel_id(const u8 **cursor, size_t *max);
+			     const struct short_channel_id *short_channel_id);
+void fromwire_short_channel_id(const u8 **cursor, size_t *max,
+			       struct short_channel_id *short_channel_id);
 
 #endif /* LIGHTNING_BITCOIN_SHORT_CHANNEL_ID_H */

bitcoin/signature.c

@@ -8,7 +8,7 @@
 #include <bitcoin/signature.h>
 #include <bitcoin/tx.h>
 #include <ccan/mem/mem.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <secp256k1_schnorrsig.h>
 #include <wire/wire.h>
 
@@ -94,7 +94,12 @@ static bool sig_has_low_r(const secp256k1_ecdsa_signature* sig)
 	return compact_sig[0] < 0x80;
 }
 
-bool dev_no_signature_grind = false;
+#if DEVELOPER
+/* Some of the spec test vectors assume no sig grinding. */
+extern bool dev_no_grind;
+
+bool dev_no_grind = false;
+#endif
 
 void sign_hash(const struct privkey *privkey,
 	       const struct sha256_double *h,
@@ -109,10 +114,11 @@ void sign_hash(const struct privkey *privkey,
 					  s,
 					  h->sha.u.u8,
 					  privkey->secret.data, NULL,
-					  dev_no_signature_grind ? NULL
-					  : extra_entropy);
+					  IFDEV(dev_no_grind ? NULL
+						: extra_entropy,
+						extra_entropy));
 		((u32 *)extra_entropy)[0]++;
-		if (dev_no_signature_grind)
+		if (IFDEV(dev_no_grind, false))
 			break;
 	} while (!sig_has_low_r(s));
 
@@ -324,7 +330,7 @@ bool signature_from_der(const u8 *der, size_t len, struct bitcoin_signature *sig
 	return true;
 }
 
-char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx, const secp256k1_ecdsa_signature *sig)
+char *fmt_signature(const tal_t *ctx, const secp256k1_ecdsa_signature *sig)
 {
 	u8 der[72];
 	size_t len = 72;
@@ -334,15 +340,17 @@ char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx, const secp256k1_ecdsa_sign
 
 	return tal_hexstr(ctx, der, len);
 }
+REGISTER_TYPE_TO_STRING(secp256k1_ecdsa_signature, fmt_signature);
 
-char *fmt_bitcoin_signature(const tal_t *ctx,
-			    const struct bitcoin_signature *sig)
+static char *bitcoin_signature_to_hexstr(const tal_t *ctx,
+					 const struct bitcoin_signature *sig)
 {
 	u8 der[73];
 	size_t len = signature_to_der(der, sig);
 
 	return tal_hexstr(ctx, der, len);
 }
+REGISTER_TYPE_TO_STRING(bitcoin_signature, bitcoin_signature_to_hexstr);
 
 void fromwire_bitcoin_signature(const u8 **cursor, size_t *max,
 				struct bitcoin_signature *sig)
@@ -376,6 +384,8 @@ char *fmt_bip340sig(const tal_t *ctx, const struct bip340sig *bip340sig)
 	return tal_hexstr(ctx, bip340sig->u8, sizeof(bip340sig->u8));
 }
 
+REGISTER_TYPE_TO_HEXSTR(bip340sig);
+
 /* BIP-340:
  *
  * This proposal suggests to include the tag by prefixing the hashed

bitcoin/signature.h

@@ -150,11 +150,8 @@ void fromwire_bip340sig(const u8 **cursor, size_t *max,
 			struct bip340sig *bip340sig);
 
 /* Get a hex string sig */
-char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx,
-				    const secp256k1_ecdsa_signature *sig);
+char *fmt_signature(const tal_t *ctx, const secp256k1_ecdsa_signature *sig);
 char *fmt_bip340sig(const tal_t *ctx, const struct bip340sig *bip340sig);
-char *fmt_bitcoin_signature(const tal_t *ctx,
-			    const struct bitcoin_signature *sig);
 
 /* For caller convenience, we hand in tag in parts (any can be "") */
 void bip340_sighash_init(struct sha256_ctx *sctx,
@@ -162,7 +159,4 @@ void bip340_sighash_init(struct sha256_ctx *sctx,
 			 const char *tag2,
 			 const char *tag3);
 
-/* Some of the spec test vectors assume no sig grinding. */
-extern bool dev_no_signature_grind;
-
 #endif /* LIGHTNING_BITCOIN_SIGNATURE_H */

bitcoin/test/run-bitcoin_block_from_hex.c

@@ -16,9 +16,6 @@ bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
 /* Generated stub for amount_sat */
 struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 { fprintf(stderr, "amount_sat called!\n"); abort(); }
@@ -27,12 +24,18 @@ struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 				       struct amount_sat a UNNEEDED,
 				       struct amount_sat b UNNEEDED)
 { fprintf(stderr, "amount_sat_add called!\n"); abort(); }
+/* Generated stub for amount_sat_div */
+struct amount_sat amount_sat_div(struct amount_sat sat UNNEEDED, u64 div UNNEEDED)
+{ fprintf(stderr, "amount_sat_div called!\n"); abort(); }
 /* Generated stub for amount_sat_eq */
 bool amount_sat_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
 { fprintf(stderr, "amount_sat_eq called!\n"); abort(); }
 /* Generated stub for amount_sat_greater_eq */
 bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
 { fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); }
+/* Generated stub for amount_sat_mul */
+bool amount_sat_mul(struct amount_sat *res UNNEEDED, struct amount_sat sat UNNEEDED, u64 mul UNNEEDED)
+{ fprintf(stderr, "amount_sat_mul called!\n"); abort(); }
 /* Generated stub for amount_sat_sub */
  bool amount_sat_sub(struct amount_sat *val UNNEEDED,
 				       struct amount_sat a UNNEEDED,

bitcoin/test/run-psbt-from-tx.c

@@ -15,9 +15,6 @@ bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
 /* Generated stub for amount_sat */
 struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 { fprintf(stderr, "amount_sat called!\n"); abort(); }
@@ -26,12 +23,18 @@ struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 				       struct amount_sat a UNNEEDED,
 				       struct amount_sat b UNNEEDED)
 { fprintf(stderr, "amount_sat_add called!\n"); abort(); }
+/* Generated stub for amount_sat_div */
+struct amount_sat amount_sat_div(struct amount_sat sat UNNEEDED, u64 div UNNEEDED)
+{ fprintf(stderr, "amount_sat_div called!\n"); abort(); }
 /* Generated stub for amount_sat_eq */
 bool amount_sat_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
 { fprintf(stderr, "amount_sat_eq called!\n"); abort(); }
 /* Generated stub for amount_sat_greater_eq */
 bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
 { fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); }
+/* Generated stub for amount_sat_mul */
+bool amount_sat_mul(struct amount_sat *res UNNEEDED, struct amount_sat sat UNNEEDED, u64 mul UNNEEDED)
+{ fprintf(stderr, "amount_sat_mul called!\n"); abort(); }
 /* Generated stub for amount_sat_sub */
  bool amount_sat_sub(struct amount_sat *val UNNEEDED,
 				       struct amount_sat a UNNEEDED,
@@ -65,9 +68,6 @@ u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEED
 /* Generated stub for sha256_double */
 void sha256_double(struct sha256_double *shadouble UNNEEDED, const void *p UNNEEDED, size_t len UNNEEDED)
 { fprintf(stderr, "sha256_double called!\n"); abort(); }
-/* Generated stub for signature_from_der */
-bool signature_from_der(const u8 *der UNNEEDED, size_t len UNNEEDED, struct bitcoin_signature *sig UNNEEDED)
-{ fprintf(stderr, "signature_from_der called!\n"); abort(); }
 /* Generated stub for signature_to_der */
 size_t signature_to_der(u8 der[73] UNNEEDED, const struct bitcoin_signature *sig UNNEEDED)
 { fprintf(stderr, "signature_to_der called!\n"); abort(); }
@@ -114,7 +114,6 @@ int main(int argc, char *argv[])
 	const struct wally_map_item *final_scriptsig = wally_map_get_integer(&tx2->psbt->inputs[0].psbt_fields, /* PSBT_IN_FINAL_SCRIPTSIG */ 0x07);
 	assert(final_scriptsig->value_len > 0);
 	assert(tx2->psbt->inputs[0].final_witness != NULL);
-	audit_psbt(tx2->psbt, tx2->psbt);
 
 	common_shutdown();
 	return 0;

bitcoin/test/run-tx-bitcoin_tx_2of2_input_witness_weight.c

@@ -19,17 +19,20 @@ bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
 /* Generated stub for amount_sat_add */
  bool amount_sat_add(struct amount_sat *val UNNEEDED,
 				       struct amount_sat a UNNEEDED,
 				       struct amount_sat b UNNEEDED)
 { fprintf(stderr, "amount_sat_add called!\n"); abort(); }
+/* Generated stub for amount_sat_div */
+struct amount_sat amount_sat_div(struct amount_sat sat UNNEEDED, u64 div UNNEEDED)
+{ fprintf(stderr, "amount_sat_div called!\n"); abort(); }
 /* Generated stub for amount_sat_greater_eq */
 bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
 { fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); }
+/* Generated stub for amount_sat_mul */
+bool amount_sat_mul(struct amount_sat *res UNNEEDED, struct amount_sat sat UNNEEDED, u64 mul UNNEEDED)
+{ fprintf(stderr, "amount_sat_mul called!\n"); abort(); }
 /* Generated stub for amount_sat_sub */
  bool amount_sat_sub(struct amount_sat *val UNNEEDED,
 				       struct amount_sat a UNNEEDED,
@@ -42,7 +45,7 @@ struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u
 struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED)
 { fprintf(stderr, "amount_tx_fee called!\n"); abort(); }
 /* Generated stub for clone_psbt */
-struct wally_psbt *clone_psbt(const tal_t *ctx UNNEEDED, const struct wally_psbt *psbt UNNEEDED)
+struct wally_psbt *clone_psbt(const tal_t *ctx UNNEEDED, struct wally_psbt *psbt UNNEEDED)
 { fprintf(stderr, "clone_psbt called!\n"); abort(); }
 /* Generated stub for fromwire */
 const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED)

bitcoin/test/run-tx-encode.c

@@ -17,9 +17,6 @@ bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
 /* Generated stub for amount_sat */
 struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 { fprintf(stderr, "amount_sat called!\n"); abort(); }
@@ -28,12 +25,18 @@ struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 				       struct amount_sat a UNNEEDED,
 				       struct amount_sat b UNNEEDED)
 { fprintf(stderr, "amount_sat_add called!\n"); abort(); }
+/* Generated stub for amount_sat_div */
+struct amount_sat amount_sat_div(struct amount_sat sat UNNEEDED, u64 div UNNEEDED)
+{ fprintf(stderr, "amount_sat_div called!\n"); abort(); }
 /* Generated stub for amount_sat_eq */
 bool amount_sat_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
 { fprintf(stderr, "amount_sat_eq called!\n"); abort(); }
 /* Generated stub for amount_sat_greater_eq */
 bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
 { fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); }
+/* Generated stub for amount_sat_mul */
+bool amount_sat_mul(struct amount_sat *res UNNEEDED, struct amount_sat sat UNNEEDED, u64 mul UNNEEDED)
+{ fprintf(stderr, "amount_sat_mul called!\n"); abort(); }
 /* Generated stub for amount_sat_sub */
  bool amount_sat_sub(struct amount_sat *val UNNEEDED,
 				       struct amount_sat a UNNEEDED,

bitcoin/tx.c

@@ -6,7 +6,7 @@
 #include <ccan/cast/cast.h>
 #include <ccan/str/hex/hex.h>
 #include <ccan/tal/str/str.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wally_psbt.h>
 #include <wire/wire.h>
 
@@ -302,6 +302,28 @@ void bitcoin_tx_output_set_amount(struct bitcoin_tx *tx, int outnum,
 	wally_psbt_output_set_amount(&tx->psbt->outputs[outnum], satoshis);
 }
 
+const u8 *wally_tx_output_get_script(const tal_t *ctx,
+				     const struct wally_tx_output *output)
+{
+	if (output->script == NULL) {
+		/* This can happen for coinbase transactions and pegin
+		 * transactions */
+		return NULL;
+	}
+
+	return tal_dup_arr(ctx, u8, output->script, output->script_len, 0);
+}
+
+const u8 *bitcoin_tx_output_get_script(const tal_t *ctx,
+				       const struct bitcoin_tx *tx, int outnum)
+{
+	const struct wally_tx_output *output;
+	assert(outnum < tx->wtx->num_outputs);
+	output = &tx->wtx->outputs[outnum];
+
+	return wally_tx_output_get_script(ctx, output);
+}
+
 u8 *bitcoin_tx_output_get_witscript(const tal_t *ctx, const struct bitcoin_tx *tx,
 				    int outnum)
 {
@@ -403,7 +425,8 @@ void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
 	assert(innum < tx->wtx->num_inputs);
 
 	in = &tx->wtx->inputs[innum];
-	CROSS_TYPE_ASSIGNMENT(&in->txhash, &outpoint->txid);
+	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+	memcpy(in->txhash, &outpoint->txid, sizeof(struct bitcoin_txid));
 	in->index = outpoint->n;
 }
 
@@ -411,13 +434,15 @@ void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
 void wally_tx_input_get_txid(const struct wally_tx_input *in,
 			     struct bitcoin_txid *txid)
 {
-	CROSS_TYPE_ASSIGNMENT(txid, &in->txhash);
+	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+	memcpy(txid, in->txhash, sizeof(struct bitcoin_txid));
 }
 
 void wally_tx_input_get_outpoint(const struct wally_tx_input *in,
 				 struct bitcoin_outpoint *outpoint)
 {
-	wally_tx_input_get_txid(in, &outpoint->txid);
+	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+	memcpy(&outpoint->txid, in->txhash, sizeof(struct bitcoin_txid));
 	outpoint->n = in->index;
 }
 
@@ -548,13 +573,9 @@ void bitcoin_tx_finalize(struct bitcoin_tx *tx)
 	assert(bitcoin_tx_check(tx));
 }
 
-struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt TAKES)
+struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt STEALS)
 {
 	size_t locktime;
-
-	if (!taken(psbt))
-		psbt = clone_psbt(tmpctx, psbt);
-
 	wally_psbt_get_locktime(psbt, &locktime);
 	struct bitcoin_tx *tx = bitcoin_tx(ctx, chainparams,
 					   psbt->num_inputs,
@@ -631,8 +652,8 @@ static struct wally_tx *pull_wtx(const tal_t *ctx,
 	return wtx;
 }
 
-struct bitcoin_tx *pull_bitcoin_tx_only(const tal_t *ctx, const u8 **cursor,
-					size_t *max)
+struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx, const u8 **cursor,
+				   size_t *max)
 {
 	struct bitcoin_tx *tx = tal(ctx, struct bitcoin_tx);
 	tx->wtx = pull_wtx(tx, cursor, max);
@@ -641,23 +662,13 @@ struct bitcoin_tx *pull_bitcoin_tx_only(const tal_t *ctx, const u8 **cursor,
 
 	tal_add_destructor(tx, bitcoin_tx_destroy);
 	tx->chainparams = chainparams;
-	tx->psbt = NULL;
+	tx->psbt = new_psbt(tx, tx->wtx);
+	if (!tx->psbt)
+		return tal_free(tx);
 
 	return tx;
 }
 
-struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx, const u8 **cursor,
-				   size_t *max)
-{
-	struct bitcoin_tx *tx = pull_bitcoin_tx_only(ctx, cursor, max);
-	if (tx) {
-		tx->psbt = new_psbt(tx, tx->wtx);
-		if (!tx->psbt)
-			return tal_free(tx);
-        }
-	return tx;
-}
-
 struct bitcoin_tx *bitcoin_tx_from_hex(const tal_t *ctx, const char *hex,
 				       size_t hexlen)
 {
@@ -694,6 +705,18 @@ fail:
 	return NULL;
 }
 
+/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
+static void reverse_bytes(u8 *arr, size_t len)
+{
+	unsigned int i;
+
+	for (i = 0; i < len / 2; i++) {
+		unsigned char tmp = arr[i];
+		arr[i] = arr[len - 1 - i];
+		arr[len - 1 - i] = tmp;
+	}
+}
+
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid)
 {
@@ -719,7 +742,7 @@ char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx)
 	return s;
 }
 
-char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
+static char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
 {
 	char *hexstr = tal_arr(ctx, char, hex_str_size(sizeof(*txid)));
 
@@ -727,15 +750,15 @@ char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
 	return hexstr;
 }
 
-char *fmt_bitcoin_outpoint(const tal_t *ctx,
-			   const struct bitcoin_outpoint *outpoint)
+static char *fmt_bitcoin_outpoint(const tal_t *ctx,
+				  const struct bitcoin_outpoint *outpoint)
 {
 	return tal_fmt(ctx, "%s:%u",
 		       fmt_bitcoin_txid(tmpctx, &outpoint->txid),
 		       outpoint->n);
 }
 
-char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
+static char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
 {
 	u8 *lin = linearize_wtx(ctx, tx);
 	char *s = tal_hex(ctx, lin);
@@ -743,15 +766,10 @@ char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
 	return s;
 }
 
-char *fmt_sha256(const tal_t *ctx, const struct sha256 *sha256)
-{
-	return tal_hexstr(ctx, sha256, sizeof(*sha256));
-}
-
-char *fmt_ripemd160(const tal_t *ctx, const struct ripemd160 *ripemd160)
-{
-	return tal_hexstr(ctx, ripemd160, sizeof(*ripemd160));
-}
+REGISTER_TYPE_TO_STRING(bitcoin_tx, fmt_bitcoin_tx);
+REGISTER_TYPE_TO_STRING(bitcoin_txid, fmt_bitcoin_txid);
+REGISTER_TYPE_TO_STRING(bitcoin_outpoint, fmt_bitcoin_outpoint);
+REGISTER_TYPE_TO_STRING(wally_tx, fmt_wally_tx);
 
 void fromwire_bitcoin_txid(const u8 **cursor, size_t *max,
 			   struct bitcoin_txid *txid)
@@ -766,13 +784,15 @@ struct bitcoin_tx *fromwire_bitcoin_tx(const tal_t *ctx,
 
 	u32 len = fromwire_u32(cursor, max);
 	size_t start = *max;
-	tx = pull_bitcoin_tx_only(ctx, cursor, max);
+	tx = pull_bitcoin_tx(ctx, cursor, max);
 	if (!tx)
 		return fromwire_fail(cursor, max);
 	// Check that we consumed len bytes
 	if (start - *max != len)
 		return fromwire_fail(cursor, max);
 
+	/* pull_bitcoin_tx sets the psbt */
+	tal_free(tx->psbt);
 	tx->psbt = fromwire_wally_psbt(tx, cursor, max);
 	if (!tx->psbt)
 		return fromwire_fail(cursor, max);
@@ -813,7 +833,8 @@ bool wally_tx_input_spends(const struct wally_tx_input *input,
 	/* Useful, as tx_part can have some NULL inputs */
 	if (!input)
 		return false;
-	if (!CROSS_TYPE_EQ(&outpoint->txid, &input->txhash))
+	BUILD_ASSERT(sizeof(outpoint->txid) == sizeof(input->txhash));
+	if (memcmp(&outpoint->txid, input->txhash, sizeof(outpoint->txid)) != 0)
 		return false;
 	return input->index == outpoint->n;
 }
@@ -897,8 +918,7 @@ size_t bitcoin_tx_input_sig_weight(void)
 /* Input weight */
 size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight)
 {
-	/* We assume < 253 witness elements */
-	size_t weight = 1 + witness_weight;
+	size_t weight = witness_weight;
 
 	/* Input weight: txid + index + sequence */
 	weight += (32 + 4 + 4) * 4;
@@ -917,32 +937,17 @@ size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight)
 	return weight;
 }
 
-size_t bitcoin_tx_input_witness_weight(enum utxotype utxotype)
+size_t bitcoin_tx_simple_input_witness_weight(void)
 {
-	switch (utxotype) {
-	case UTXO_P2SH_P2WPKH:
-	case UTXO_P2WPKH:
-		/* Account for witness (sig + key) */
-		return bitcoin_tx_input_sig_weight() + 1 + 33;
-	case UTXO_P2WSH_FROM_CLOSE:
-		/* BOLT #3:
-		 * #### `to_remote` Output
-		 *
-		 * If `option_anchors` applies to the commitment
-		 * transaction, the `to_remote` output is encumbered by a one
-		 * block csv lock.
-		 *    <remotepubkey> OP_CHECKSIGVERIFY 1 OP_CHECKSEQUENCEVERIFY
-		 *
-		 * The output is spent by an input with `nSequence` field set
-		 * to `1` and witness: <remote_sig>
-		 * Otherwise, this output is a simple P2WPKH to `remotepubkey`.
-		 */
-		/* In practice, these predate anchors, so: */
-		return 1 + 1 + bitcoin_tx_input_sig_weight();
-	case UTXO_P2TR:
-		return 1 + 64;
-	}
-	abort();
+	/* Account for witness (1 byte count + sig + key) */
+	return 1 + (bitcoin_tx_input_sig_weight() + 1 + 33);
+}
+
+/* We only do segwit inputs, and we assume witness is sig + key  */
+size_t bitcoin_tx_simple_input_weight(bool p2sh)
+{
+	return bitcoin_tx_input_weight(p2sh,
+				       bitcoin_tx_simple_input_witness_weight());
 }
 
 size_t bitcoin_tx_2of2_input_witness_weight(void)
@@ -961,19 +966,17 @@ size_t bitcoin_tx_2of2_input_witness_weight(void)
 		);
 }
 
-size_t change_weight(void)
-{
-	return bitcoin_tx_output_weight(chainparams->is_elements ? BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN : BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
-}
-
 struct amount_sat change_fee(u32 feerate_perkw,	size_t total_weight)
 {
+	size_t outweight;
 	struct amount_sat fee;
 
 	/* Must be able to pay for its own additional weight */
+	outweight = bitcoin_tx_output_weight(chainparams->is_elements ? BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN : BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
+
 	/* Rounding can cause off by one errors, so we do this */
 	if (!amount_sat_sub(&fee,
-			    amount_tx_fee(feerate_perkw, change_weight() + total_weight),
+			    amount_tx_fee(feerate_perkw, outweight + total_weight),
 			    amount_tx_fee(feerate_perkw, total_weight)))
 		abort();
 	return fee;
@@ -996,15 +999,11 @@ struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw,
 
 u32 tx_feerate(const struct bitcoin_tx *tx)
 {
-
-	u32 feerate;
+	struct amount_sat fee = bitcoin_tx_compute_fee(tx);
 
 	/* Fee should not overflow! */
-	if (!amount_feerate(&feerate,
-			    bitcoin_tx_compute_fee(tx),
-			    bitcoin_tx_weight(tx))) {
+	if (!amount_sat_mul(&fee, fee, 1000))
 		abort();
-	}
 
-	return feerate;
+	return amount_sat_div(fee, bitcoin_tx_weight(tx)).satoshis; /* Raw: txfee */
 }

bitcoin/tx.h

@@ -14,9 +14,7 @@
 /* BIP 125: Any nsequence < 0xFFFFFFFE is replacable.
  * And bitcoind uses this value. */
 #define BITCOIN_TX_RBF_SEQUENCE 0xFFFFFFFD
-
 struct wally_psbt;
-struct ripemd160;
 
 struct bitcoin_txid {
 	struct sha256_double shad;
@@ -48,17 +46,6 @@ struct bitcoin_tx_output {
 	u8 *script;
 };
 
-enum utxotype {
-	/* Obsolete: we used to have P2SH-wrapped outputs (removed in 24.02, though can still have old UTXOs) */
-	UTXO_P2SH_P2WPKH = 1,
-	/* "bech32" addresses */
-	UTXO_P2WPKH = 2,
-	/* Used for closing addresses: implies ->close_info is non-NULL */
-	UTXO_P2WSH_FROM_CLOSE = 3,
-	/* "p2tr" addresses. */
-	UTXO_P2TR = 4,
-};
-
 struct bitcoin_tx_output *new_tx_output(const tal_t *ctx,
 					struct amount_sat amount,
 					const u8 *script);
@@ -90,18 +77,6 @@ struct bitcoin_tx *clone_bitcoin_tx(const tal_t *ctx,
 struct bitcoin_tx *bitcoin_tx_from_hex(const tal_t *ctx, const char *hex,
 				       size_t hexlen);
 
-/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
-static inline void reverse_bytes(u8 *arr, size_t len)
-{
-	unsigned int i;
-
-	for (i = 0; i < len / 2; i++) {
-		unsigned char tmp = arr[i];
-		arr[i] = arr[len - 1 - i];
-		arr[len - 1 - i] = tmp;
-	}
-}
-
 /* Parse hex string to get txid (reversed, a-la bitcoind). */
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid);
@@ -111,18 +86,12 @@ bool bitcoin_txid_to_hex(const struct bitcoin_txid *txid,
 			 char *hexstr, size_t hexstr_len);
 
 /* Create a bitcoin_tx from a psbt */
-struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx,
-					struct wally_psbt *psbt TAKES);
+struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt);
 
 /* Internal de-linearization functions. */
-/* Pull a bitcoin tx, and create a PSBT wrapper for it */
 struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx,
 				   const u8 **cursor, size_t *max);
 
-/* Pull a bitcoin tx without creating a PSBT wrapper for it */
-struct bitcoin_tx *pull_bitcoin_tx_only(const tal_t *ctx,
-					const u8 **cursor, size_t *max);
-
 /* Helper to create a wally_tx_output: make sure to wally_tx_output_free!
  * Returns NULL if amount is extreme (wally doesn't like).
  */
@@ -170,6 +139,25 @@ wally_tx_output_get_amount(const struct wally_tx_output *output);
 void bitcoin_tx_output_set_amount(struct bitcoin_tx *tx, int outnum,
 				  struct amount_sat amount);
 
+/**
+ * Helper to get the script of a script's output as a tal_arr
+ *
+ * Internally we use a `wally_tx` to represent the transaction. The script
+ * attached to a `wally_tx_output` is not a `tal_arr`, so in order to keep the
+ * comfort of being able to call `tal_bytelen` and similar on a script we just
+ * return a `tal_arr` clone of the original script.
+ */
+const u8 *bitcoin_tx_output_get_script(const tal_t *ctx, const struct bitcoin_tx *tx, int outnum);
+
+/**
+ * Helper to get the script of a script's output as a tal_arr
+ *
+ * The script attached to a `wally_tx_output` is not a `tal_arr`, so in order to keep the
+ * comfort of being able to call `tal_bytelen` and similar on a script we just
+ * return a `tal_arr` clone of the original script.
+ */
+const u8 *wally_tx_output_get_script(const tal_t *ctx,
+				     const struct wally_tx_output *output);
 /**
  * Helper to get a witness script for an output.
  */
@@ -311,14 +299,6 @@ void towire_bitcoin_outpoint(u8 **pptr, const struct bitcoin_outpoint *outp);
 void fromwire_bitcoin_outpoint(const u8 **cursor, size_t *max,
 			       struct bitcoin_outpoint *outp);
 char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx);
-char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid);
-char *fmt_bitcoin_outpoint(const tal_t *ctx,
-			   const struct bitcoin_outpoint *outpoint);
-char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx);
-
-/* For want of somewhere better to define them! */
-char *fmt_sha256(const tal_t *ctx, const struct sha256 *sha256);
-char *fmt_ripemd160(const tal_t *ctx, const struct ripemd160 *ripemd160);
 
 /* Various weights of transaction parts. */
 size_t bitcoin_tx_core_weight(size_t num_inputs, size_t num_outputs);
@@ -327,22 +307,18 @@ size_t bitcoin_tx_output_weight(size_t outscript_len);
 /* Weight to push sig on stack. */
 size_t bitcoin_tx_input_sig_weight(void);
 
-/* Segwit input, but with parameter for witness weight (size).
- * witness_weight must include the varint_size() for each witness element,
- * but not the varint_size() for the number of elements. */
+/* Segwit input, but with parameter for witness weight (size) */
 size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight);
 
-/* The witness weight */
-size_t bitcoin_tx_input_witness_weight(enum utxotype utxotype);
+/* The witness weight for a simple (sig + key) input */
+size_t bitcoin_tx_simple_input_witness_weight(void);
+
+/* We only do segwit inputs, and we assume witness is sig + key  */
+size_t bitcoin_tx_simple_input_weight(bool p2sh);
 
 /* The witness for our 2of2 input (closing or commitment tx). */
 size_t bitcoin_tx_2of2_input_witness_weight(void);
 
-/**
- * change_weight - what's the weight of a change output?
- */
-size_t change_weight(void);
-
 /**
  * change_fee - what's the cost to add a change output to this tx?
  * @feerate_perkw: feerate.

bitcoin/tx_parts.c

@@ -10,11 +10,69 @@ static void destroy_wally_tx_input(struct wally_tx_input *in)
 	wally_tx_input_free(in);
 }
 
+static struct wally_tx_input *clone_input(const struct wally_tx_input *src)
+{
+	struct wally_tx_input *in;
+	int ret;
+
+	if (is_elements(chainparams)) {
+		ret = wally_tx_elements_input_init_alloc
+			(src->txhash, sizeof(src->txhash),
+			 src->index, src->sequence,
+			 src->script, src->script_len,
+			 src->witness,
+			 src->blinding_nonce, sizeof(src->blinding_nonce),
+			 src->entropy, sizeof(src->entropy),
+			 src->issuance_amount, src->issuance_amount_len,
+			 src->inflation_keys, src->inflation_keys_len,
+			 src->issuance_amount_rangeproof,
+			 src->issuance_amount_rangeproof_len,
+			 src->inflation_keys_rangeproof,
+			 src->inflation_keys_rangeproof_len,
+			 src->pegin_witness,
+			 &in);
+	} else {
+		ret = wally_tx_input_init_alloc(src->txhash, sizeof(src->txhash),
+						src->index, src->sequence,
+						src->script, src->script_len,
+						src->witness, &in);
+	}
+	assert(ret == WALLY_OK);
+
+	tal_add_destructor(in, destroy_wally_tx_input);
+	return in;
+}
+
 static void destroy_wally_tx_output(struct wally_tx_output *out)
 {
 	wally_tx_output_free(out);
 }
 
+static struct wally_tx_output *clone_output(const struct wally_tx_output *src)
+{
+	struct wally_tx_output *out;
+	int ret;
+
+	if (is_elements(chainparams)) {
+		ret = wally_tx_elements_output_init_alloc
+			(src->script, src->script_len,
+			 src->asset, src->asset_len,
+			 src->value, src->value_len,
+			 src->nonce, src->nonce_len,
+			 src->surjectionproof, src->surjectionproof_len,
+			 src->rangeproof, src->rangeproof_len,
+			 &out);
+	} else {
+		ret = wally_tx_output_init_alloc(src->satoshi,
+						 src->script, src->script_len,
+						 &out);
+	}
+	assert(ret == WALLY_OK);
+
+	tal_add_destructor(out, destroy_wally_tx_output);
+	return out;
+}
+
 struct tx_parts *tx_parts_from_wally_tx(const tal_t *ctx,
 					const struct wally_tx *wtx,
 					int input, int output)
@@ -29,19 +87,13 @@ struct tx_parts *tx_parts_from_wally_tx(const tal_t *ctx,
 	for (size_t i = 0; i < wtx->num_inputs; i++) {
 		if (input != -1 && input != i)
 			continue;
-		if (wally_tx_input_clone_alloc(&wtx->inputs[i],
-					       &txp->inputs[i]) != WALLY_OK)
-			abort();
-		tal_add_destructor(txp->inputs[i], destroy_wally_tx_input);
+		txp->inputs[i] = clone_input(&wtx->inputs[i]);
 	}
 
 	for (size_t i = 0; i < wtx->num_outputs; i++) {
 		if (output != -1 && output != i)
 			continue;
-		if (wally_tx_output_clone_alloc(&wtx->outputs[i],
-						&txp->outputs[i]) != WALLY_OK)
-			abort();
-		tal_add_destructor(txp->outputs[i], destroy_wally_tx_output);
+		txp->outputs[i] = clone_output(&wtx->outputs[i]);
 
 		/* Cheat a bit by also setting the numeric satoshi
 		 * value, otherwise we end up converting a

ccan/README

@@ -1,3 +1,3 @@
 CCAN imported from http://ccodearchive.net.
 
-CCAN version: init-2593-gca094039
+CCAN version: init-2577-g1ae4c432

ccan/ccan/bitmap/bitmap.c

@@ -9,7 +9,7 @@
 #define BIT_ALIGN_DOWN(n)	((n) & ~(BITMAP_WORD_BITS - 1))
 #define BIT_ALIGN_UP(n)		BIT_ALIGN_DOWN((n) + BITMAP_WORD_BITS - 1)
 
-void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m)
+void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
 	unsigned long am = BIT_ALIGN_DOWN(m);
@@ -19,22 +19,22 @@ void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m)
 	assert(m >= n);
 
 	if (am < an) {
-		BITMAP_WORD(b, n) &= ~bitmap_bswap(headmask & tailmask);
+		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask & tailmask);
 		return;
 	}
 
 	if (an > n)
-		BITMAP_WORD(b, n) &= ~bitmap_bswap(headmask);
+		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask);
 
 	if (am > an)
-		memset(&BITMAP_WORD(b, an), 0,
+		memset(&BITMAP_WORD(bitmap, an), 0,
 		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
 
 	if (m > am)
-		BITMAP_WORD(b, m) &= ~bitmap_bswap(tailmask);
+		BITMAP_WORD(bitmap, m) &= ~bitmap_bswap(tailmask);
 }
 
-void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m)
+void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
 	unsigned long am = BIT_ALIGN_DOWN(m);
@@ -44,19 +44,19 @@ void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m)
 	assert(m >= n);
 
 	if (am < an) {
-		BITMAP_WORD(b, n) |= bitmap_bswap(headmask & tailmask);
+		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask & tailmask);
 		return;
 	}
 
 	if (an > n)
-		BITMAP_WORD(b, n) |= bitmap_bswap(headmask);
+		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask);
 
 	if (am > an)
-		memset(&BITMAP_WORD(b, an), 0xff,
+		memset(&BITMAP_WORD(bitmap, an), 0xff,
 		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
 
 	if (m > am)
-		BITMAP_WORD(b, m) |= bitmap_bswap(tailmask);
+		BITMAP_WORD(bitmap, m) |= bitmap_bswap(tailmask);
 }
 
 static int bitmap_clz(bitmap_word w)
@@ -76,7 +76,7 @@ static int bitmap_clz(bitmap_word w)
 #endif
 }
 
-unsigned long bitmap_ffs(const bitmap *b,
+unsigned long bitmap_ffs(const bitmap *bitmap,
 			 unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
@@ -87,7 +87,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	assert(m >= n);
 
 	if (am < an) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, n));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
 
 		w &= (headmask & tailmask);
 
@@ -95,7 +95,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	if (an > n) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, n));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
 
 		w &= headmask;
 
@@ -104,7 +104,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	while (an < am) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, an));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, an));
 
 		if (w)
 			return an + bitmap_clz(w);
@@ -113,7 +113,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	if (m > am) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, m));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, m));
 
 		w &= tailmask;
 

ccan/ccan/bitmap/bitmap.h

@@ -58,37 +58,37 @@ static inline bitmap_word bitmap_bswap(bitmap_word w)
 #define BITMAP_TAIL(_bm, _nbits) \
 	(BITMAP_TAILWORD(_bm, _nbits) & BITMAP_TAILBITS(_nbits))
 
-static inline void bitmap_set_bit(bitmap *b, unsigned long n)
+static inline void bitmap_set_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) |= BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) |= BITMAP_WORDBIT(n);
 }
 
-static inline void bitmap_clear_bit(bitmap *b, unsigned long n)
+static inline void bitmap_clear_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) &= ~BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) &= ~BITMAP_WORDBIT(n);
 }
 
-static inline void bitmap_change_bit(bitmap *b, unsigned long n)
+static inline void bitmap_change_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) ^= BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) ^= BITMAP_WORDBIT(n);
 }
 
-static inline bool bitmap_test_bit(const bitmap *b, unsigned long n)
+static inline bool bitmap_test_bit(const bitmap *bitmap, unsigned long n)
 {
-	return !!(BITMAP_WORD(b, n) & BITMAP_WORDBIT(n));
+	return !!(BITMAP_WORD(bitmap, n) & BITMAP_WORDBIT(n));
 }
 
-void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m);
-void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m);
+void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m);
+void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m);
 
-static inline void bitmap_zero(bitmap *b, unsigned long nbits)
+static inline void bitmap_zero(bitmap *bitmap, unsigned long nbits)
 {
-	memset(b, 0, bitmap_sizeof(nbits));
+	memset(bitmap, 0, bitmap_sizeof(nbits));
 }
 
-static inline void bitmap_fill(bitmap *b, unsigned long nbits)
+static inline void bitmap_fill(bitmap *bitmap, unsigned long nbits)
 {
-	memset(b, 0xff, bitmap_sizeof(nbits));
+	memset(bitmap, 0xff, bitmap_sizeof(nbits));
 }
 
 static inline void bitmap_copy(bitmap *dst, const bitmap *src,
@@ -161,36 +161,37 @@ static inline bool bitmap_subset(const bitmap *src1, const bitmap *src2,
 	return true;
 }
 
-static inline bool bitmap_full(const bitmap *b, unsigned long nbits)
+static inline bool bitmap_full(const bitmap *bitmap, unsigned long nbits)
 {
 	unsigned long i;
 
 	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
-		if (b[i].w != -1UL)
+		if (bitmap[i].w != -1UL)
 			return false;
 	}
 	if (BITMAP_HASTAIL(nbits) &&
-	    (BITMAP_TAIL(b, nbits) != BITMAP_TAILBITS(nbits)))
+	    (BITMAP_TAIL(bitmap, nbits) != BITMAP_TAILBITS(nbits)))
 		return false;
 
 	return true;
 }
 
-static inline bool bitmap_empty(const bitmap *b, unsigned long nbits)
+static inline bool bitmap_empty(const bitmap *bitmap, unsigned long nbits)
 {
 	unsigned long i;
 
 	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
-		if (b[i].w != 0)
+		if (bitmap[i].w != 0)
 			return false;
 	}
-	if (BITMAP_HASTAIL(nbits) && (BITMAP_TAIL(b, nbits) != 0))
+	if (BITMAP_HASTAIL(nbits) && (BITMAP_TAIL(bitmap, nbits) != 0))
 		return false;
 
 	return true;
 }
 
-unsigned long bitmap_ffs(const bitmap *b, unsigned long n, unsigned long m);
+unsigned long bitmap_ffs(const bitmap *bitmap,
+			 unsigned long n, unsigned long m);
 
 /*
  * Allocation functions
@@ -220,26 +221,26 @@ static inline bitmap *bitmap_alloc1(unsigned long nbits)
 	return bitmap;
 }
 
-static inline bitmap *bitmap_realloc0(bitmap *b,
+static inline bitmap *bitmap_realloc0(bitmap *bitmap,
 				      unsigned long obits, unsigned long nbits)
 {
-	b = realloc(b, bitmap_sizeof(nbits));
+	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
 
-	if ((nbits > obits) && b)
-		bitmap_zero_range(b, obits, nbits);
+	if ((nbits > obits) && bitmap)
+		bitmap_zero_range(bitmap, obits, nbits);
 
-	return b;
+	return bitmap;
 }
 
-static inline bitmap *bitmap_realloc1(bitmap *b,
+static inline bitmap *bitmap_realloc1(bitmap *bitmap,
 				      unsigned long obits, unsigned long nbits)
 {
-	b = realloc(b, bitmap_sizeof(nbits));
+	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
 
-	if ((nbits > obits) && b)
-		bitmap_fill_range(b, obits, nbits);
+	if ((nbits > obits) && bitmap)
+		bitmap_fill_range(bitmap, obits, nbits);
 
-	return b;
+	return bitmap;
 }
 
 #endif /* CCAN_BITMAP_H_ */

ccan/ccan/fdpass/fdpass.c

@@ -3,7 +3,6 @@
 #include <sys/socket.h>
 #include <errno.h>
 #include <string.h>
-#include <sys/types.h>
 
 bool fdpass_send(int sockout, int fd)
 {

ccan/ccan/htable/htable_type.h

@@ -1,23 +1,19 @@
 /* Licensed under LGPLv2+ - see LICENSE file for details */
 #ifndef CCAN_HTABLE_TYPE_H
 #define CCAN_HTABLE_TYPE_H
-#include "config.h"
-#include <assert.h>
 #include <ccan/htable/htable.h>
 #include <ccan/compiler/compiler.h>
+#include "config.h"
 
 /**
- * HTABLE_DEFINE_NODUPS_TYPE/HTABLE_DEFINE_DUPS_TYPE - create a set of htable ops for a type
+ * HTABLE_DEFINE_TYPE - create a set of htable ops for a type
  * @type: a type whose pointers will be values in the hash.
  * @keyof: a function/macro to extract a key: <keytype> @keyof(const type *elem)
  * @hashfn: a hash function for a @key: size_t @hashfn(const <keytype> *)
  * @eqfn: an equality function keys: bool @eqfn(const type *, const <keytype> *)
  * @prefix: a prefix for all the functions to define (of form <name>_*)
  *
- * There are two variants, one of which allows duplicate keys, and one which
- * does not.  The defined functions differ in some cases, as shown below.
- *
- * NULL values may not be placed into the hash table (nor (void *)1).
+ * NULL values may not be placed into the hash table.
  *
  * This defines the type hashtable type and an iterator type:
  *	struct <name>;
@@ -37,18 +33,15 @@
  *
  * Delete and delete-by key return true if it was in the set:
  *	bool <name>_del(struct <name> *ht, const <type> *e);
- *	bool <name>_delkey(struct <name> *ht, const <keytype> *k) (NODUPS only);
+ *	bool <name>_delkey(struct <name> *ht, const <keytype> *k);
  *
  * Delete by iterator:
  *	bool <name>_delval(struct <name> *ht, struct <name>_iter *i);
  *
- * Find and return the matching element, or NULL:
- *	type *<name>_get(const struct @name *ht, const <keytype> *k) (NODUPS only);
+ * Find and return the (first) matching element, or NULL:
+ *	type *<name>_get(const struct @name *ht, const <keytype> *k);
  *
- * Test for an element:
- *	bool <name>_exists(const struct @name *ht, const <keytype> *k);
- *
- * Find and return all matching elements, or NULL (DUPS only):
+ * Find and return all matching elements, or NULL:
  *	type *<name>_getfirst(const struct @name *ht, const <keytype> *k,
  *			      struct <name>_iter *i);
  *	type *<name>_getnext(const struct @name *ht, const <keytype> *k,
@@ -66,7 +59,7 @@
  * You can use HTABLE_INITIALIZER like so:
  *	struct <name> ht = { HTABLE_INITIALIZER(ht.raw, <name>_hash, NULL) };
  */
-#define HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
+#define HTABLE_DEFINE_TYPE(type, keyof, hashfn, eqfn, name)		\
 	struct name { struct htable raw; };				\
 	struct name##_iter { struct htable_iter i; };			\
 	static inline size_t name##_hash(const void *elem, void *priv)	\
@@ -96,33 +89,66 @@
 	{								\
 		return htable_copy(&dst->raw, &src->raw);		\
 	}								\
+	static inline bool name##_add(struct name *ht, const type *elem) \
+	{								\
+		return htable_add(&ht->raw, hashfn(keyof(elem)), elem);	\
+	}								\
 	static inline UNNEEDED bool name##_del(struct name *ht,		\
 					       const type *elem)	\
 	{								\
 		return htable_del(&ht->raw, hashfn(keyof(elem)), elem);	\
 	}								\
-	static inline UNNEEDED type *name##_getmatch_(const struct name *ht, \
-				         const HTABLE_KTYPE(keyof, type) k, \
-				         size_t h,			\
-				         type *v,			\
-					 struct htable_iter *iter)	\
-	{								\
-		while (v) {						\
-			if (eqfn(v, k))					\
-				break;					\
-			v = htable_nextval(&ht->raw, iter, h);		\
-		}							\
-		return v;						\
-	}								\
-	static inline UNNEEDED bool name##_exists(const struct name *ht,	\
+	static inline UNNEEDED type *name##_get(const struct name *ht,	\
 				       const HTABLE_KTYPE(keyof, type) k) \
 	{								\
 		struct htable_iter i;					\
 		size_t h = hashfn(k);					\
-		void *v;						\
+		void *c;						\
 									\
-		v = htable_firstval(&ht->raw, &i, h);			\
-		return name##_getmatch_(ht, k, h, v, &i) != NULL;	\
+		for (c = htable_firstval(&ht->raw,&i,h);		\
+		     c;							\
+		     c = htable_nextval(&ht->raw,&i,h)) {		\
+			if (eqfn(c, k))					\
+				return c;				\
+		}							\
+		return NULL;						\
+	}								\
+	static inline UNNEEDED type *name##_getmatch_(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+				         size_t h,			\
+				         type *v,			\
+					 struct name##_iter *iter)	\
+	{								\
+		while (v) {						\
+			if (eqfn(v, k))					\
+				break;					\
+			v = htable_nextval(&ht->raw, &iter->i, h);	\
+		}							\
+		return v;						\
+	}								\
+	static inline UNNEEDED type *name##_getfirst(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+					 struct name##_iter *iter)	\
+	{								\
+		size_t h = hashfn(k);					\
+		type *v = htable_firstval(&ht->raw, &iter->i, h);	\
+		return name##_getmatch_(ht, k, h, v, iter);			\
+	}								\
+	static inline UNNEEDED type *name##_getnext(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+					 struct name##_iter *iter)	\
+	{								\
+		size_t h = hashfn(k);					\
+		type *v = htable_nextval(&ht->raw, &iter->i, h);	\
+		return name##_getmatch_(ht, k, h, v, iter);		\
+	}								\
+	static inline UNNEEDED bool name##_delkey(struct name *ht,	\
+					 const HTABLE_KTYPE(keyof, type) k) \
+	{								\
+		type *elem = name##_get(ht, k);				\
+		if (elem)						\
+			return name##_del(ht, elem);			\
+		return false;						\
 	}								\
 	static inline UNNEEDED void name##_delval(struct name *ht,	\
 						  struct name##_iter *iter) \
@@ -151,64 +177,6 @@
 		return htable_prev(&ht->raw, &iter->i);			\
 	}
 
-#define HTABLE_DEFINE_NODUPS_TYPE(type, keyof, hashfn, eqfn, name)	\
-	HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
-	static inline UNNEEDED type *name##_get(const struct name *ht,	\
-				       const HTABLE_KTYPE(keyof, type) k) \
-	{								\
-		struct htable_iter i;					\
-		size_t h = hashfn(k);					\
-		void *v;						\
-									\
-		v = htable_firstval(&ht->raw, &i, h);			\
-		return name##_getmatch_(ht, k, h, v, &i);			\
-	}								\
-	static inline bool name##_add(struct name *ht, const type *elem) \
-	{								\
-		/* Open-coded for slightly more efficiency */		\
-		const HTABLE_KTYPE(keyof, type) k = keyof(elem);	\
-		struct htable_iter i;					\
-		size_t h = hashfn(k);					\
-		void *v;						\
-									\
-		v = htable_firstval(&ht->raw, &i, h);			\
-		assert(!name##_getmatch_(ht, k, h, v, &i));		\
-		return htable_add(&ht->raw, h, elem);			\
-	}								\
-	static inline UNNEEDED bool name##_delkey(struct name *ht,	\
-					 const HTABLE_KTYPE(keyof, type) k) \
-	{								\
-		type *elem = name##_get(ht, k);				\
-		if (elem)						\
-			return name##_del(ht, elem);			\
-		return false;						\
-	}
-
-#define HTABLE_DEFINE_DUPS_TYPE(type, keyof, hashfn, eqfn, name)	\
-	HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
-	static inline bool name##_add(struct name *ht, const type *elem) \
-	{								\
-		const HTABLE_KTYPE(keyof, type) k = keyof(elem);	\
-		return htable_add(&ht->raw, hashfn(k), elem);		\
-	}								\
-	static inline UNNEEDED type *name##_getfirst(const struct name *ht, \
-				         const HTABLE_KTYPE(keyof, type) k, \
-					 struct name##_iter *iter)	\
-	{								\
-		size_t h = hashfn(k);					\
-		type *v = htable_firstval(&ht->raw, &iter->i, h);	\
-		return name##_getmatch_(ht, k, h, v, &iter->i);		\
-	}								\
-	static inline UNNEEDED type *name##_getnext(const struct name *ht, \
-				         const HTABLE_KTYPE(keyof, type) k, \
-					 struct name##_iter *iter)	\
-	{								\
-		size_t h = hashfn(k);					\
-		type *v = htable_nextval(&ht->raw, &iter->i, h);	\
-		return name##_getmatch_(ht, k, h, v, &iter->i);		\
-	}
-
-
 #if HAVE_TYPEOF
 #define HTABLE_KTYPE(keyof, type) typeof(keyof((const type *)NULL))
 #else

ccan/ccan/htable/test/run-type-int.c

@@ -38,7 +38,7 @@ static bool cmp(const struct obj *obj, const unsigned int key)
 	return obj->key == key;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
+HTABLE_DEFINE_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
 
 static void add_vals(struct htable_obj *ht,
 		     struct obj val[], unsigned int num)
@@ -112,19 +112,14 @@ static bool check_mask(struct htable *ht, const struct obj val[], unsigned num)
 	return true;
 }
 
-/* This variant allows duplicates! */
-HTABLE_DEFINE_DUPS_TYPE(struct obj, objkey, objhash, cmp, htable_obj_dups);
-
 int main(void)
 {
 	unsigned int i;
 	struct htable_obj ht, ht2;
-	struct htable_obj_dups ht_dups;
 	struct obj val[NUM_VALS], *result;
 	unsigned int dne;
 	void *p;
 	struct htable_obj_iter iter;
-	struct htable_obj_dups_iter dups_iter;
 
 	plan_tests(29);
 	for (i = 0; i < NUM_VALS; i++)
@@ -188,35 +183,32 @@ int main(void)
 	del_vals_bykey(&ht, val, NUM_VALS);
 	del_vals_bykey(&ht2, val, NUM_VALS);
 
-	/* Duplicates-allowed tests */
-	htable_obj_dups_init(&ht_dups);
 	/* Write two of the same value. */
 	val[1] = val[0];
-	htable_obj_dups_add(&ht_dups, &val[0]);
-	htable_obj_dups_add(&ht_dups, &val[1]);
+	htable_obj_add(&ht, &val[0]);
+	htable_obj_add(&ht, &val[1]);
 	i = 0;
 
-	result = htable_obj_dups_getfirst(&ht_dups, i, &dups_iter);
+	result = htable_obj_getfirst(&ht, i, &iter);
 	ok1(result == &val[0] || result == &val[1]);
 	if (result == &val[0]) {
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[0]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[0]));
+		ok1(htable_obj_getfirst(&ht, i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 	} else {
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[1]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[1]));
+		ok1(htable_obj_getfirst(&ht, i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 	}
 
-	htable_obj_dups_clear(&ht_dups);
 	htable_obj_clear(&ht);
 	htable_obj_clear(&ht2);
 	return exit_status();

ccan/ccan/htable/test/run-type.c

@@ -33,10 +33,7 @@ static bool cmp(const struct obj *obj, const unsigned int *key)
 	return obj->key == *key;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(struct obj, objkey, objhash, cmp,
-			  htable_obj);
-HTABLE_DEFINE_DUPS_TYPE(struct obj, objkey, objhash, cmp,
-			htable_obj_dups);
+HTABLE_DEFINE_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
 
 static void add_vals(struct htable_obj *ht,
 		     struct obj val[], unsigned int num)
@@ -114,14 +111,12 @@ int main(void)
 {
 	unsigned int i;
 	struct htable_obj ht, ht2;
-	struct htable_obj_dups ht_dups;
 	struct obj val[NUM_VALS], *result;
 	unsigned int dne;
 	void *p;
 	struct htable_obj_iter iter;
-	struct htable_obj_dups_iter dups_iter;
 
-	plan_tests(36);
+	plan_tests(35);
 	for (i = 0; i < NUM_VALS; i++)
 		val[i].key = i;
 	dne = i;
@@ -187,37 +182,32 @@ int main(void)
 	del_vals_bykey(&ht, val, NUM_VALS);
 	del_vals_bykey(&ht2, val, NUM_VALS);
 
-	/* Duplicates-allowed tests */
-	htable_obj_dups_init(&ht_dups);
-
 	/* Write two of the same value. */
 	val[1] = val[0];
-	htable_obj_dups_add(&ht_dups, &val[0]);
-	htable_obj_dups_add(&ht_dups, &val[1]);
+	htable_obj_add(&ht, &val[0]);
+	htable_obj_add(&ht, &val[1]);
 	i = 0;
 
-	result = htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter);
+	result = htable_obj_getfirst(&ht, &i, &iter);
 	ok1(result == &val[0] || result == &val[1]);
 	if (result == &val[0]) {
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[0]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[0]));
+		ok1(htable_obj_getfirst(&ht, &i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 	} else {
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[1]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[1]));
+		ok1(htable_obj_getfirst(&ht, &i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 	}
 
-	htable_obj_dups_clear(&ht_dups);
-	ok1(htable_obj_dups_count(&ht_dups) == 0);
 	htable_obj_clear(&ht);
 	ok1(htable_obj_count(&ht) == 0);
 	htable_obj_clear(&ht2);

ccan/ccan/htable/tools/density.c

@@ -26,7 +26,7 @@ static bool cmp(const ptrint_t *p, uintptr_t k)
 	return key(p) == k;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(ptrint_t, key, hash_uintptr, cmp, htable_ptrint);
+HTABLE_DEFINE_TYPE(ptrint_t, key, hash_uintptr, cmp, htable_ptrint);
 
 /* Nanoseconds per operation */
 static size_t normalize(const struct timeabs *start,

ccan/ccan/htable/tools/speed.c

@@ -33,7 +33,7 @@ static bool cmp(const struct object *object, const unsigned int *key)
 	return object->key == *key;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(struct object, objkey, hash_obj, cmp, htable_obj);
+HTABLE_DEFINE_TYPE(struct object, objkey, hash_obj, cmp, htable_obj);
 
 static unsigned int popcount(unsigned long val)
 {

ccan/ccan/htable/tools/stringspeed.c

@@ -31,7 +31,7 @@ static bool cmp(const char *obj, const char *key)
 	return strcmp(obj, key) == 0;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(char, strkey, hash_str, cmp, htable_str);
+HTABLE_DEFINE_TYPE(char, strkey, hash_str, cmp, htable_str);
 
 /* Nanoseconds per operation */
 static size_t normalize(const struct timeabs *start,

ccan/ccan/intmap/benchmark/speed.c

@@ -59,7 +59,7 @@ static bool eqfn(const struct htable_elem *elem, const uint64_t index)
 {
 	return elem->index == index;
 }
-HTABLE_DEFINE_NODUPS_TYPE(struct htable_elem, keyof, hashfn, eqfn, hash);
+HTABLE_DEFINE_TYPE(struct htable_elem, keyof, hashfn, eqfn, hash);
 
 static bool check_val(intmap_index_t i, uint64_t *v, uint64_t *expected)
 {

ccan/ccan/io/benchmarks/Makefile

@@ -1,17 +1,16 @@
-ALL:=run-loop run-different-speed run-length-prefix run-stream-many
+ALL:=run-loop run-different-speed run-length-prefix
 CCANDIR:=../../..
 CFLAGS:=-Wall -I$(CCANDIR) -O3 -flto
 LDFLAGS:=-O3 -flto
 LDLIBS:=-lrt
 
-OBJS:=time.o poll.o io.o err.o timer.o list.o ccan-tal.o ccan-take.o ccan-ilog.o
+OBJS:=time.o poll.o io.o err.o timer.o list.o
 
 default: $(ALL)
 
 run-loop: run-loop.o $(OBJS)
 run-different-speed: run-different-speed.o $(OBJS)
 run-length-prefix: run-length-prefix.o $(OBJS)
-run-stream-many: run-stream-many.o $(OBJS)
 
 time.o: $(CCANDIR)/ccan/time/time.c
 	$(CC) $(CFLAGS) -c -o $@ $<
@@ -25,12 +24,6 @@ io.o: $(CCANDIR)/ccan/io/io.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 err.o: $(CCANDIR)/ccan/err/err.c
 	$(CC) $(CFLAGS) -c -o $@ $<
-ccan-ilog.o: $(CCANDIR)/ccan/ilog/ilog.c
-	$(CC) $(CFLAGS) -c -o $@ $<
-ccan-tal.o: $(CCANDIR)/ccan/tal/tal.c
-	$(CC) $(CFLAGS) -c -o $@ $<
-ccan-take.o: $(CCANDIR)/ccan/take/take.c
-	$(CC) $(CFLAGS) -c -o $@ $<
 
 clean:
 	rm -f *.o $(ALL)

ccan/ccan/io/benchmarks/run-stream-many.c

@@ -1,132 +0,0 @@
-/* Wait for many fds to connect, then try to stream the file to some of them in small chunks.
- *
- * This approximates the connectd behaviour in CLN, where we send gossip to peers.
- */
-#include <ccan/io/io.h>
-#include <ccan/ptrint/ptrint.h>
-#include <ccan/time/time.h>
-#include <inttypes.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-#include <err.h>
-#include <signal.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-
-/* We expect num_expected connections, and how many will be writers */
-static size_t max_readers, max_writers;
-
-/* How many raeaders and writers still going */
-static size_t num_readers, num_writers;
-
-/* How many times to do the write */
-static size_t write_iterations;
-
-/* The buffer to write */
-static char writebuf[256];
-
-/* We need this for readers, though we don't actually care! */
-static size_t len_ignored;
-
-struct timemono start_time;
-
-static void finished(void)
-{
-	struct timerel elapsed = timemono_since(start_time);
-	printf("Finished: %"PRIu64"usec\n", time_to_usec(elapsed));
-	exit(0);
-}
-
-static struct io_plan *write_loop(struct io_conn *conn, ptrint_t *iter)
-{
-	ptrdiff_t n = ptr2int(iter);
-
-	if (n > write_iterations) {
-		--num_writers;
-		if (num_writers == 0)
-			finished();
-		return io_wait(conn, conn, io_never, NULL);
-	}
-	return io_write(conn, writebuf, sizeof(writebuf), write_loop, int2ptr(n + 1));
-}
-
-static struct io_plan *read_loop(struct io_conn *conn, void *unused)
-{
-	return io_read_partial(conn, writebuf, sizeof(writebuf), &len_ignored, read_loop, unused);
-}
-
-static void reader_failed(struct io_conn *conn, intptr_t *num)
-{
-	err(1, "Reader %zu/%zu", (size_t)ptr2int(num), max_readers);
-}
-
-static void writer_failed(struct io_conn *conn, intptr_t *num)
-{
-	err(1, "Writer %zu/%zu", (size_t)ptr2int(num), max_writers);
-}
-
-static struct io_plan *connection_in(struct io_conn *conn, void *sleep_on)
-{
-	if (num_readers < max_readers) {
-		printf("r");
-		fflush(stdout);
-		num_readers++;
-		io_set_finish(conn, reader_failed, int2ptr(num_readers));
-		return read_loop(conn, NULL);
-	}
-
-	/* We assign writers last: not sure it matters, but it's more reflective
-	 * of lightning where more recent connections tend to ask for gossip */
-	num_writers++;
-	printf("w");
-	fflush(stdout);
-
-	io_set_finish(conn, writer_failed, int2ptr(num_writers));
-	io_set_finish(conn, writer_failed, NULL);
-	if (num_writers < max_writers)
-		return io_wait(conn, sleep_on, write_loop, int2ptr(0));
-
-	/* Everyone is connected.  Wake them and start final one */
-	io_wake(sleep_on);
-	printf("Starting!\n");
-	start_time = time_mono();
-	return write_loop(conn, int2ptr(0));
-}
-
-int main(int argc, char *argv[])
-{
-	int fd;
-	struct sockaddr_in s4;
-	int on = 1;
-
-	if (argc != 5)
-		errx(1, "Usage: <portnum> <num-idle> <num-streaming> <mb-streamed>");
-
-	memset(&s4, 0, sizeof(s4));
-	s4.sin_family = AF_INET;
-	s4.sin_port = htons(atol(argv[1]));
-	s4.sin_addr.s_addr = INADDR_ANY;
-
-	max_readers = atol(argv[2]);
-	max_writers = atol(argv[3]);
-	write_iterations = atol(argv[4]) * (1024 * 1024 / sizeof(writebuf));
-
-	fd = socket(AF_INET, SOCK_STREAM, 0);
-	if (fd < 0)
-		err(1, "Creating socket");
-
-	/* Re-use, please.. */
-	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)))
-		err(1, "Setting reuseaddr");
-
-	if (bind(fd, &s4, sizeof(s4)) != 0)
-		err(1, "Binding");
-
-	if (listen(fd, 1) != 0)
-		err(1, "Listening");
-
-	io_new_listener(NULL, fd, connection_in, &s4);
-	io_loop(NULL, NULL);
-	errx(1, "Sockets exited?");
-}

ccan/ccan/io/fdpass/fdpass.c

@@ -4,12 +4,6 @@
 #include <ccan/io/io_plan.h>
 #include <errno.h>
 
-static void destroy_conn_close_send_fd(struct io_conn *conn,
-				       struct io_plan_arg *arg)
-{
-	close(arg->u1.s);
-}
-
 static int do_fd_send(int fd, struct io_plan_arg *arg)
 {
 	if (!fdpass_send(fd, arg->u1.s)) {
@@ -18,11 +12,8 @@ static int do_fd_send(int fd, struct io_plan_arg *arg)
 			return 0;
 		return -1;
 	}
-	if (arg->u2.vp) {
-		struct io_conn *conn = arg->u2.vp;
+	if (arg->u2.s)
 		close(arg->u1.s);
-		tal_del_destructor2(conn, destroy_conn_close_send_fd, arg);
-	}
 	return 1;
 }
 
@@ -35,11 +26,7 @@ struct io_plan *io_send_fd_(struct io_conn *conn,
 	struct io_plan_arg *arg = io_plan_arg(conn, IO_OUT);
 
 	arg->u1.s = fd;
-	/* We need conn ptr for destructor */
-	arg->u2.vp = fdclose ? conn : NULL;
-	/* If conn closes before sending, we still need to close fd */
-	if (fdclose)
-		tal_add_destructor2(conn, destroy_conn_close_send_fd, arg);
+	arg->u2.s = fdclose;
 
 	return io_set_plan(conn, IO_OUT, do_fd_send, next, next_arg);
 }
@@ -52,9 +39,7 @@ static int do_fd_recv(int fd, struct io_plan_arg *arg)
 		/* In case ccan/io ever gets smart with non-blocking. */
 		if (errno == EAGAIN || errno == EWOULDBLOCK)
 			return 0;
-		/* If they can't handle the error, this will close conn! */
-		if (!io_get_extended_errors())
-			return -1;
+		return -1;
 	}
 	*(int *)arg->u1.vp = fdin;
 	return 1;

ccan/ccan/io/fdpass/fdpass.h

@@ -45,9 +45,8 @@ struct io_plan *io_send_fd_(struct io_conn *conn,
  * @arg: @next argument
  *
  * This creates a plan to receive a file descriptor, as sent by
- * io_send_fd.  Once it's all read, the @next function will be called.
- * On an error, if io_get_extended_errors() is true, then @next is called
- * and @fd will be -1, otherwise the finish function is called.
+ * io_send_fd.  Once it's all read, the @next function will be called:
+ * on an error, the finish function is called instead.
  *
  * Note that the I/O may actually be done immediately.
  *

ccan/ccan/io/io.c

@@ -15,7 +15,6 @@
 void *io_loop_return;
 
 struct io_plan io_conn_freed;
-static bool io_extended_errors;
 
 struct io_listener *io_new_listener_(const tal_t *ctx, int fd,
 				     struct io_plan *(*init)(struct io_conn *,
@@ -384,19 +383,9 @@ void io_wake(const void *wait)
 	backend_wake(wait);
 }
 
-enum plan_result {
-	/* Destroyed, do not touch */
-	FREED,
-	/* Worked, call again. */
-	KEEP_GOING,
-	/* Failed with EAGAIN or did partial. */
-	EXHAUSTED,
-	/* No longer interested in read (or write) */
-	UNINTERESTED
-};
-
-static enum plan_result do_plan(struct io_conn *conn, struct io_plan *plan,
-				bool idle_on_epipe)
+/* Returns false if this should not be touched (eg. freed). */
+static bool do_plan(struct io_conn *conn, struct io_plan *plan,
+		    bool idle_on_epipe)
 {
 	/* We shouldn't have polled for this event if this wasn't true! */
 	assert(plan->status == IO_POLLING_NOTSTARTED
@@ -404,26 +393,18 @@ static enum plan_result do_plan(struct io_conn *conn, struct io_plan *plan,
 
 	switch (plan->io(conn->fd.fd, &plan->arg)) {
 	case -1:
-		/* This is expected, as we call optimistically! */
-		if (errno == EAGAIN)
-			return EXHAUSTED;
 		if (errno == EPIPE && idle_on_epipe) {
 			plan->status = IO_UNSET;
 			backend_new_plan(conn);
-			return UNINTERESTED;
+			return false;
 		}
 		io_close(conn);
-		return FREED;
+		return false;
 	case 0:
 		plan->status = IO_POLLING_STARTED;
-		/* If it started but didn't finish, don't call again. */
-		return EXHAUSTED;
+		return true;
 	case 1:
-		if (!next_plan(conn, plan))
-			return FREED;
-		if (plan->status == IO_POLLING_NOTSTARTED)
-			return KEEP_GOING;
-		return UNINTERESTED;
+		return next_plan(conn, plan);
 	default:
 		/* IO should only return -1, 0 or 1 */
 		abort();
@@ -432,43 +413,16 @@ static enum plan_result do_plan(struct io_conn *conn, struct io_plan *plan,
 
 void io_ready(struct io_conn *conn, int pollflags)
 {
-	enum plan_result res;
+	if (pollflags & POLLIN)
+		if (!do_plan(conn, &conn->plan[IO_IN], false))
+			return;
 
-	if (pollflags & POLLIN) {
-		for (;;) {
-			res = do_plan(conn, &conn->plan[IO_IN], false);
-			switch (res) {
-			case FREED:
-				return;
-			case EXHAUSTED:
-			case UNINTERESTED:
-				goto try_write;
-			case KEEP_GOING:
-				continue;
-			}
-			abort();
-		}
-	}
-
-try_write:
-	if (pollflags & POLLOUT) {
-		for (;;) {
-			/* If we're writing to a closed pipe, we need to wait for
-			 * read to fail if we're duplex: we want to drain it! */
-			res = do_plan(conn, &conn->plan[IO_OUT],
-				      conn->plan[IO_IN].status == IO_POLLING_NOTSTARTED
-				      || conn->plan[IO_IN].status == IO_POLLING_STARTED);
-			switch (res) {
-			case FREED:
-			case EXHAUSTED:
-			case UNINTERESTED:
-				return;
-			case KEEP_GOING:
-				continue;
-			}
-			abort();
-		}
-	}
+	if (pollflags & POLLOUT)
+		/* If we're writing to a closed pipe, we need to wait for
+		 * read to fail if we're duplex: we want to drain it! */
+		do_plan(conn, &conn->plan[IO_OUT],
+			conn->plan[IO_IN].status == IO_POLLING_NOTSTARTED
+			|| conn->plan[IO_IN].status == IO_POLLING_STARTED);
 }
 
 void io_do_always(struct io_plan *plan)
@@ -586,7 +540,7 @@ struct io_plan *io_sock_shutdown(struct io_conn *conn)
 	/* And leave unset .*/
 	return &conn->plan[IO_IN];
 }
-
+	
 bool io_flush_sync(struct io_conn *conn)
 {
 	struct io_plan *plan = &conn->plan[IO_OUT];
@@ -622,13 +576,3 @@ again:
 	io_fd_block(io_conn_fd(conn), false);
 	return ok;
 }
-
-void io_set_extended_errors(bool state)
-{
-	io_extended_errors = state;
-}
-
-bool io_get_extended_errors(void)
-{
-	return io_extended_errors;
-}

ccan/ccan/io/io.h

@@ -59,7 +59,7 @@ struct io_conn;
 	io_new_conn_((ctx), (fd),					\
 		     typesafe_cb_preargs(struct io_plan *, void *,	\
 					 (init), (arg),			\
-					 struct io_conn *),		\
+					 struct io_conn *conn),		\
 		     (void *)(arg))
 
 struct io_conn *io_new_conn_(const tal_t *ctx, int fd,
@@ -113,8 +113,6 @@ void io_set_finish_(struct io_conn *conn,
  * (tal'ocated off @ctx) and pass that to init().  Note that if there is
  * an error on this file descriptor, it will be freed.
  *
- * Note: if the accept fails (usually due to EMFILE), init() will be called
- * wth
  * Returns NULL on error (and sets errno).
  *
  * Example:
@@ -825,13 +823,4 @@ int (*io_poll_override(int (*poll)(struct pollfd *fds, nfds_t nfds, int timeout)
  */
 const void *io_have_fd(int fd, bool *listener);
 
-/**
- * io_set_extended_errors - enable callbacks for errors.
- * @state: true or false.
- *
- * Defaults false for compatibility.  See io_new_conn for what this changes.
- */
-void io_set_extended_errors(bool state);
-bool io_get_extended_errors(void);
-
 #endif /* CCAN_IO_H */

ccan/ccan/io/poll.c

@@ -270,12 +270,10 @@ static void accept_conn(struct io_listener *l)
 {
 	int fd = accept(l->fd.fd, NULL, NULL);
 
-	if (fd < 0) {
-		/* If they've enabled it, this is how we tell them */
-		if (io_get_extended_errors())
-			l->init(NULL, l->arg);
+	/* FIXME: What to do here? */
+	if (fd < 0)
 		return;
-	}
+
 	io_new_conn(l->ctx, fd, l->init, l->arg);
 }
 

ccan/ccan/io/test/run-43-io_plan_in_started.c

@@ -18,17 +18,9 @@ static struct io_plan *init_in_conn(struct io_conn *conn, char *buf)
 	return io_read(conn, buf, 2, in_conn_done, NULL);
 }
 
-/* Every second time we say we're exhausted */
 static int do_nothing(int fd, struct io_plan_arg *arg)
 {
-	static bool read_once;
-
-	read_once = !read_once;
-	if (read_once)
-		return 1;
-
-	errno = EAGAIN;
-	return -1;
+	return 1;
 }
 
 static struct io_plan *dummy_write(struct io_conn *conn,

ccan/ccan/likely/likely.c

@@ -29,8 +29,8 @@ static bool trace_eq(const struct trace *t1, const struct trace *t2)
 }
 
 /* struct thash */
-HTABLE_DEFINE_NODUPS_TYPE(struct trace, (const struct trace *), hash_trace, trace_eq,
-			  thash);
+HTABLE_DEFINE_TYPE(struct trace, (const struct trace *), hash_trace, trace_eq,
+		   thash);
 
 static struct thash htable
 = { HTABLE_INITIALIZER(htable.raw, thash_hash, NULL) };

ccan/ccan/opt/opt.h

@@ -530,7 +530,7 @@ struct opt_table *opt_find_short(char arg);
  * You can set bits in type e.g. (1<<OPT_USER_START) to (1<<OPT_USER_END)
  * when calling _opt_register. */
 #define OPT_USER_START 8
-#define OPT_USER_END 30
+#define OPT_USER_END 15
 
 /* Below here are private declarations. */
 /* You can use this directly to build tables, but the macros will ensure
@@ -540,7 +540,7 @@ enum opt_type {
 	OPT_HASARG = 2,		/* -f arg|--foo=arg|--foo arg */
 	OPT_SUBTABLE = 4,	/* Actually, longopt points to a subtable... */
 	OPT_EARLY = 8,		/* Parse this from opt_early_parse() only. */
-	OPT_END = 31,		/* End of the table. */
+	OPT_END = 16,		/* End of the table. */
 
 	/* Make sure no compiler will assume we never have large
 	 * values in the enum! */

ccan/ccan/rune/rune.c

@@ -345,8 +345,6 @@ static const char *rune_alt_single(const tal_t *ctx,
 				 memmem(fieldval_str, fieldval_strlen,
 					alt->value, strlen(alt->value)));
 	case RUNE_COND_INT_LESS:
-		if (!fieldval_str && !fieldval_int)
-			return tal_fmt(ctx, "%s not present", alt->fieldname);
 		err = integer_compare_valid(ctx, fieldval_int,
 					    alt, &runeval_int);
 		if (err)
@@ -354,8 +352,6 @@ static const char *rune_alt_single(const tal_t *ctx,
 		return cond_test(ctx, alt, "is greater or equal to",
 				 *fieldval_int < runeval_int);
 	case RUNE_COND_INT_GREATER:
-		if (!fieldval_str && !fieldval_int)
-			return tal_fmt(ctx, "%s not present", alt->fieldname);
 		err = integer_compare_valid(ctx, fieldval_int,
 					    alt, &runeval_int);
 		if (err)

ccan/ccan/tal/str/str.c

@@ -14,14 +14,21 @@
 
 char *tal_strdup_(const tal_t *ctx, const char *p, const char *label)
 {
-	return tal_dup_arr_label(ctx, char, p, strlen(p) + 1, 0, label);
+	/* We have to let through NULL for take(). */
+	return tal_dup_arr_label(ctx, char, p, p ? strlen(p) + 1: 1, 0, label);
 }
 
 char *tal_strndup_(const tal_t *ctx, const char *p, size_t n, const char *label)
 {
-	size_t len = strnlen(p, n);
+	size_t len;
 	char *ret;
 
+	/* We have to let through NULL for take(). */
+	if (likely(p))
+		len = strnlen(p, n);
+	else
+		len = n;
+
 	ret = tal_dup_arr_label(ctx, char, p, len, 1, label);
 	if (ret)
 		ret[len] = '\0';
@@ -77,6 +84,9 @@ char *tal_vfmt_(const tal_t *ctx, const char *fmt, va_list ap, const char *label
 {
 	char *buf;
 
+	if (!fmt && taken(fmt))
+		return NULL;
+
 	/* A decent guess to start. */
 	buf = tal_arr_label(ctx, char, strlen(fmt) * 2, label);
 	if (!do_vfmt(&buf, 0, fmt, ap))
@@ -86,6 +96,9 @@ char *tal_vfmt_(const tal_t *ctx, const char *fmt, va_list ap, const char *label
 
 bool tal_append_vfmt(char **baseptr, const char *fmt, va_list ap)
 {
+	if (!fmt && taken(fmt))
+		return false;
+
 	return do_vfmt(baseptr, strlen(*baseptr), fmt, ap);
 }
 
@@ -107,7 +120,13 @@ char *tal_strcat_(const tal_t *ctx, const char *s1, const char *s2,
 	size_t len1, len2;
 	char *ret;
 
-	len1 = strlen(s1);
+	if (unlikely(!s2) && taken(s2)) {
+		if (taken(s1))
+			tal_free(s1);
+		return NULL;
+	}
+	/* We have to let through NULL for take(). */
+	len1 = s1 ? strlen(s1) : 0;
 	len2 = strlen(s2);
 
 	ret = tal_dup_arr_label(ctx, char, s1, len1, len2 + 1, label);
@@ -132,11 +151,13 @@ char **tal_strsplit_(const tal_t *ctx,
 			tal_free(string);
 		if (taken(delims))
 			tal_free(delims);
-		return parts;
+		return NULL;
 	}
 	str = tal_strdup(parts, string);
 	if (unlikely(!str))
 		goto fail;
+	if (unlikely(!delims) && is_taken(delims))
+		goto fail;
 
 	if (flags == STR_NO_EMPTY)
 		str += strspn(str, delims);
@@ -164,14 +185,10 @@ char **tal_strsplit_(const tal_t *ctx,
 	return parts;
 
 fail:
-#ifdef CCAN_TAL_NEVER_RETURN_NULL
-	abort();
-#else
 	tal_free(parts);
 	if (taken(delims))
 		tal_free(delims);
 	return NULL;
-#endif
 }
 
 char *tal_strjoin_(const tal_t *ctx,
@@ -182,6 +199,12 @@ char *tal_strjoin_(const tal_t *ctx,
 	char *ret = NULL;
 	size_t totlen = 0, dlen;
 
+	if (unlikely(!strings) && is_taken(strings))
+		goto fail;
+
+	if (unlikely(!delim) && is_taken(delim))
+		goto fail;
+
 	dlen = strlen(delim);
 	ret = tal_arr_label(ctx, char, dlen*2+1, label);
 	if (!ret)
@@ -246,9 +269,15 @@ bool tal_strreg_(const tal_t *ctx, const char *string, const char *label,
 	unsigned int i;
 	va_list ap;
 
+	if (unlikely(!regex) && is_taken(regex))
+		goto fail_no_re;
+
 	if (regcomp(&r, regex, REG_EXTENDED) != 0)
 		goto fail_no_re;
 
+	if (unlikely(!string) && is_taken(string))
+		goto fail;
+
 	if (regexec(&r, string, nmatch, matches, 0) != 0)
 		goto fail;
 

ccan/ccan/tal/str/str.h

@@ -12,18 +12,17 @@
 /**
  * tal_strdup - duplicate a string
  * @ctx: NULL, or tal allocated object to be parent.
- * @p: the string to copy (can be take(), must not be NULL).
+ * @p: the string to copy (can be take()).
  *
  * The returned string will have tal_count() == strlen() + 1.
  */
 #define tal_strdup(ctx, p) tal_strdup_(ctx, p, TAL_LABEL(char, "[]"))
-char *tal_strdup_(const tal_t *ctx, const char *p TAKES, const char *label)
-	TAL_RETURN_PTR NON_NULL_ARGS(2);
+char *tal_strdup_(const tal_t *ctx, const char *p TAKES, const char *label);
 
 /**
  * tal_strndup - duplicate a limited amount of a string.
  * @ctx: NULL, or tal allocated object to be parent.
- * @p: the string to copy (can be take(), must not be NULL).
+ * @p: the string to copy (can be take()).
  * @n: the maximum length to copy.
  *
  * Always gives a nul-terminated string, with strlen() <= @n.
@@ -31,25 +30,24 @@ char *tal_strdup_(const tal_t *ctx, const char *p TAKES, const char *label)
  */
 #define tal_strndup(ctx, p, n) tal_strndup_(ctx, p, n, TAL_LABEL(char, "[]"))
 char *tal_strndup_(const tal_t *ctx, const char *p TAKES, size_t n,
-		   const char *label)
-	TAL_RETURN_PTR NON_NULL_ARGS(2);
+		   const char *label);
 
 /**
  * tal_fmt - allocate a formatted string
  * @ctx: NULL, or tal allocated object to be parent.
- * @fmt: the printf-style format (can be take(), must not be NULL).
+ * @fmt: the printf-style format (can be take()).
  *
  * The returned string will have tal_count() == strlen() + 1.
  */
 #define tal_fmt(ctx, ...)				 \
 	tal_fmt_(ctx, TAL_LABEL(char, "[]"), __VA_ARGS__)
 char *tal_fmt_(const tal_t *ctx, const char *label, const char *fmt TAKES,
-	       ...) PRINTF_FMT(3,4) TAL_RETURN_PTR NON_NULL_ARGS(3);
+	        ...) PRINTF_FMT(3,4);
 
 /**
  * tal_vfmt - allocate a formatted string (va_list version)
  * @ctx: NULL, or tal allocated object to be parent.
- * @fmt: the printf-style format (can be take(), must not be NULL).
+ * @fmt: the printf-style format (can be take()).
  * @va: the va_list containing the format args.
  *
  * The returned string will have tal_count() == strlen() + 1.
@@ -58,42 +56,40 @@ char *tal_fmt_(const tal_t *ctx, const char *label, const char *fmt TAKES,
 	tal_vfmt_(ctx, fmt, va, TAL_LABEL(char, "[]"))
 char *tal_vfmt_(const tal_t *ctx, const char *fmt TAKES, va_list ap,
 		const char *label)
-	PRINTF_FMT(2,0) TAL_RETURN_PTR NON_NULL_ARGS(2);
+	PRINTF_FMT(2,0);
 
 /**
  * tal_append_fmt - append a formatted string to a talloc string.
  * @baseptr: a pointer to the tal string to be appended to.
- * @fmt: the printf-style format (can be take(), must not be NULL).
+ * @fmt: the printf-style format (can be take()).
  *
  * Returns false on allocation failure.
  * Otherwise tal_count(*@baseptr) == strlen(*@baseptr) + 1.
  */
-bool tal_append_fmt(char **baseptr, const char *fmt TAKES, ...)
-	PRINTF_FMT(2,3) NON_NULL_ARGS(2);
+bool tal_append_fmt(char **baseptr, const char *fmt TAKES, ...) PRINTF_FMT(2,3);
 
 /**
  * tal_append_vfmt - append a formatted string to a talloc string (va_list)
  * @baseptr: a pointer to the tal string to be appended to.
- * @fmt: the printf-style format (can be take(), must not be NULL).
+ * @fmt: the printf-style format (can be take()).
  * @va: the va_list containing the format args.
  *
  * Returns false on allocation failure.
  * Otherwise tal_count(*@baseptr) == strlen(*@baseptr) + 1.
  */
-bool tal_append_vfmt(char **baseptr, const char *fmt TAKES, va_list ap)
-	NON_NULL_ARGS(2);
+bool tal_append_vfmt(char **baseptr, const char *fmt TAKES, va_list ap);
 
 /**
  * tal_strcat - join two strings together
  * @ctx: NULL, or tal allocated object to be parent.
- * @s1: the first string (can be take(), must not be NULL).
- * @s2: the second string (can be take(), must not be NULL).
+ * @s1: the first string (can be take()).
+ * @s2: the second string (can be take()).
  *
  * The returned string will have tal_count() == strlen() + 1.
  */
 #define tal_strcat(ctx, s1, s2) tal_strcat_(ctx, s1, s2, TAL_LABEL(char, "[]"))
 char *tal_strcat_(const tal_t *ctx, const char *s1 TAKES, const char *s2 TAKES,
-		  const char *label) TAL_RETURN_PTR NON_NULL_ARGS(2,3);
+		  const char *label);
 
 enum strsplit {
 	STR_EMPTY_OK,
@@ -103,8 +99,8 @@ enum strsplit {
 /**
  * tal_strsplit - Split string into an array of substrings
  * @ctx: the context to tal from (often NULL).
- * @string: the string to split (can be take(), must not be NULL).
- * @delims: delimiters where lines should be split (can be take(), must not be NULL).
+ * @string: the string to split (can be take()).
+ * @delims: delimiters where lines should be split (can be take()).
  * @flags: whether to include empty substrings.
  *
  * This function splits a single string into multiple strings.
@@ -141,8 +137,7 @@ char **tal_strsplit_(const tal_t *ctx,
 		     const char *string TAKES,
 		     const char *delims TAKES,
 		     enum strsplit flag,
-		     const char *label)
-	TAL_RETURN_PTR NON_NULL_ARGS(2,3);
+		     const char *label);
 
 enum strjoin {
 	STR_TRAIL,
@@ -152,8 +147,8 @@ enum strjoin {
 /**
  * tal_strjoin - Join an array of substrings into one long string
  * @ctx: the context to tal from (often NULL).
- * @strings: the NULL-terminated array of strings to join (can be take(), must not be NULL)
- * @delim: the delimiter to insert between the strings (can be take(), must not be NULL)
+ * @strings: the NULL-terminated array of strings to join (can be take())
+ * @delim: the delimiter to insert between the strings (can be take())
  * @flags: whether to add a delimieter to the end
  *
  * This function joins an array of strings into a single string.  The
@@ -180,14 +175,13 @@ char *tal_strjoin_(const void *ctx,
 		   char *strings[] TAKES,
 		   const char *delim TAKES,
 		   enum strjoin flags,
-		   const char *label)
-	TAL_RETURN_PTR NON_NULL_ARGS(2,3);
+		   const char *label);
 
 /**
  * tal_strreg - match/extract from a string via (extended) regular expressions.
  * @ctx: the context to tal from (often NULL)
- * @string: the string to try to match (can be take(), must not be NULL)
- * @regex: the regular expression to match (can be take(), must not be NULL)
+ * @string: the string to try to match (can be take())
+ * @regex: the regular expression to match (can be take())
  * ...: pointers to strings to allocate for subexpressions.
  *
  * Returns true if we matched, in which case any parenthesized
@@ -227,6 +221,5 @@ char *tal_strjoin_(const void *ctx,
 #define tal_strreg(ctx, string, ...)					\
 	tal_strreg_(ctx, string, TAL_LABEL(char, "[]"), __VA_ARGS__)
 bool tal_strreg_(const void *ctx, const char *string TAKES,
-		 const char *label, const char *regex TAKES, ...)
-	NON_NULL_ARGS(2,4);
+		 const char *label, const char *regex, ...);
 #endif /* CCAN_STR_TAL_H */

ccan/ccan/tal/tal.c

@@ -456,24 +456,13 @@ static void del_tree(struct tal_hdr *t, const tal_t *orig, int saved_errno)
         freefn(t);
 }
 
-/* Don't have compiler complain we're returning NULL if we promised not to! */
-static void *null_alloc_failed(void)
-{
-#ifdef CCAN_TAL_NEVER_RETURN_NULL
-	abort();
-#else
-	return NULL;
-#endif /* CCAN_TAL_NEVER_RETURN_NULL */
-}
-
 void *tal_alloc_(const tal_t *ctx, size_t size, bool clear, const char *label)
 {
         struct tal_hdr *child, *parent = debug_tal(to_tal_hdr_or_null(ctx));
 
         child = allocate(sizeof(struct tal_hdr) + size);
 	if (!child)
-		return null_alloc_failed();
-
+		return NULL;
 	if (clear)
 		memset(from_tal_hdr(child), 0, size);
         child->prop = (void *)label;
@@ -481,7 +470,7 @@ void *tal_alloc_(const tal_t *ctx, size_t size, bool clear, const char *label)
 
         if (!add_child(parent, child)) {
 		freefn(child);
-		return null_alloc_failed();
+		return NULL;
 	}
 	debug_tal(parent);
 	if (notifiers)
@@ -512,7 +501,7 @@ void *tal_alloc_arr_(const tal_t *ctx, size_t size, size_t count, bool clear,
 		     const char *label)
 {
 	if (!adjust_size(&size, count))
-		return null_alloc_failed();
+		return NULL;
 
 	return tal_alloc_(ctx, size, clear, label);
 }

ccan/ccan/tal/tal.h

@@ -11,14 +11,6 @@
 #include <stdbool.h>
 #include <stdarg.h>
 
-/* Define this for better optimization if you never override errfn
- * to something tat returns */
-#ifdef CCAN_TAL_NEVER_RETURN_NULL
-#define TAL_RETURN_PTR RETURNS_NONNULL
-#else
-#define TAL_RETURN_PTR
-#endif /* CCAN_TAL_NEVER_RETURN_NULL */
-
 /**
  * tal_t - convenient alias for void to mark tal pointers.
  *
@@ -425,8 +417,7 @@ tal_t *tal_parent(const tal_t *ctx);
  * @error_fn: called on errors or NULL (default is abort)
  *
  * The defaults are set up so tal functions never return NULL, but you
- * can override error_fn to change that.  error_fn can return (only if
- * you haven't defined CCAN_TAL_NEVER_RETURN_NULL!), and is
+ * can override erorr_fn to change that.  error_fn can return, and is
  * called if alloc_fn or resize_fn fail.
  *
  * If any parameter is NULL, that function is unchanged.
@@ -530,11 +521,9 @@ bool tal_set_name_(tal_t *ctx, const char *name, bool literal);
 #define tal_typechk_(ptr, ptype) (ptr)
 #endif
 
-void *tal_alloc_(const tal_t *ctx, size_t bytes, bool clear, const char *label)
-	TAL_RETURN_PTR;
+void *tal_alloc_(const tal_t *ctx, size_t bytes, bool clear, const char *label);
 void *tal_alloc_arr_(const tal_t *ctx, size_t bytes, size_t count, bool clear,
-		     const char *label)
-	TAL_RETURN_PTR;
+		     const char *label);
 
 void *tal_dup_(const tal_t *ctx, const void *p TAKES, size_t size,
 	       size_t n, size_t extra, bool nullok, const char *label);

ccan/ccan/tcon/tcon.h

@@ -147,7 +147,7 @@
  * It evaluates to @x so you can chain it.
  */
 #define tcon_check_ptr(x, canary, expr)				\
-	(sizeof(0 ? (expr) : &(x)->_tcon[0].canary) ? (x) : (x))
+	(sizeof((expr) ? (expr) : &(x)->_tcon[0].canary) ? (x) : (x))
 
 /**
  * tcon_type - the type within a container (or void *)

ccan/ccan/time/test/run-monotonic.c

@@ -7,13 +7,13 @@ int main(void)
 	struct timemono t1, t2;
 	struct timerel t3;
 
-	plan_tests(10);
+	plan_tests(5);
 
 	/* Test time_mono */
 	t1 = time_mono();
 	t2 = time_mono();
 
-	ok1(!timemono_before(t2, t1));
+	ok1(!time_less_(t2.ts, t1.ts));
 
 	t3.ts.tv_sec = 1;
 	t3.ts.tv_nsec = 0;
@@ -24,11 +24,5 @@ int main(void)
 	ok1(timemono_add(t1, t3).ts.tv_sec == t1.ts.tv_sec + 1);
 	ok1(timemono_add(t2, t3).ts.tv_nsec == t2.ts.tv_nsec);
 
-	ok1(timemono_sub(timemono_add(t1, t3), t3).ts.tv_sec == t1.ts.tv_sec);
-	ok1(timemono_sub(timemono_add(t1, t3), t3).ts.tv_nsec == t1.ts.tv_nsec);
-
-	ok1(timemono_after(timemono_add(t1, t3), t1));
-	ok1(!timemono_after(t1, timemono_add(t1, t3)));
-	ok1(!timemono_after(t1, t1));
 	return exit_status();
 }

ccan/ccan/time/time.h

@@ -193,23 +193,6 @@ static inline bool time_greater(struct timerel a, struct timerel b)
 	return time_greater_(a.ts, b.ts);
 }
 
-/**
- * timemono_after - is a after b?
- * @a: one monotonic time.
- * @b: another monotonic time.
- *
- * Example:
- *	static bool timed_out(const struct timemono *start)
- *	{
- *	#define TIMEOUT time_from_msec(1000)
- *		return timemono_after(time_mono(), timemono_add(*start, TIMEOUT));
- *	}
- */
-static inline bool timemono_after(struct timemono a, struct timemono b)
-{
-	return time_greater_(a.ts, b.ts);
-}
-
 static inline bool time_less_(struct timespec a, struct timespec b)
 {
 	if (TIME_CHECK(a).tv_sec < TIME_CHECK(b).tv_sec)
@@ -237,23 +220,6 @@ static inline bool time_before(struct timeabs a, struct timeabs b)
 	return time_less_(a.ts, b.ts);
 }
 
-/**
- * timemono_before - is a before b?
- * @a: one monotonic time.
- * @b: another monotonic time.
- *
- * Example:
- *	static bool still_valid(const struct timemono *start)
- *	{
- *	#define TIMEOUT time_from_msec(1000)
- *		return timemono_before(time_mono(), timemono_add(*start, TIMEOUT));
- *	}
- */
-static inline bool timemono_before(struct timemono a, struct timemono b)
-{
-	return time_less_(a.ts, b.ts);
-}
-
 /**
  * time_less - is a before b?
  * @a: one relative time.
@@ -438,29 +404,6 @@ static inline struct timeabs timeabs_sub(struct timeabs abs, struct timerel rel)
 	return t;
 }
 
-/**
- * timemono_sub - subtract a relative time from a monotonic time
- * @mono: the monotonic time.
- * @rel: the relative time.
- *
- * This returns a well formed struct timemono of @mono - @rel.
- *
- * Example:
- *	// We do one every second.
- *	static struct timemono previous_time(void)
- *	{
- *		return timemono_sub(time_mono(), time_from_msec(1000));
- *	}
- */
-static inline struct timemono timemono_sub(struct timemono mono, struct timerel rel)
-{
-	struct timemono t;
-
-	t.ts = time_sub_(mono.ts, rel.ts);
-	return t;
-}
-
-
 static inline struct timespec time_add_(struct timespec a, struct timespec b)
 {
 	struct timespec sum;
@@ -545,8 +488,6 @@ static inline struct timerel timerel_add(struct timerel a, struct timerel b)
  * @div: number to divide it by.
  *
  * Example:
- *	#include <sys/wait.h>
- *
  *	// How long does it take to do a fork?
  *	static struct timerel forking_time(void)
  *	{

channeld/Makefile

@@ -60,7 +60,6 @@ CHANNELD_COMMON_OBJS :=				\
 	common/status_wiregen.o			\
 	common/gossip_store.o			\
 	common/hmac.o				\
-	common/hsm_capable.o				\
 	common/interactivetx.o			\
 	common/htlc_state.o			\
 	common/htlc_trim.o			\
@@ -83,6 +82,7 @@ CHANNELD_COMMON_OBJS :=				\
 	common/psbt_keypath.o		\
 	common/psbt_open.o			\
 	common/psbt_internal.o			\
+	common/private_channel_announcement.o	\
 	common/pseudorand.o			\
 	common/read_peer_msg.o			\
 	common/setup.o				\
@@ -90,17 +90,20 @@ CHANNELD_COMMON_OBJS :=				\
 	common/status_wire.o			\
 	common/subdaemon.o			\
 	common/timeout.o			\
+	common/type_to_string.o			\
 	common/utils.o				\
 	common/utxo.o				\
 	common/version.o			\
 	common/wire_error.o			\
 	common/wireaddr.o			\
+	gossipd/gossipd_peerd_wiregen.o		\
+	gossipd/gossip_store_wiregen.o		\
 	wire/fromwire.o				\
 	wire/towire.o
 
 channeld/full_channel_error_names_gen.h: channeld/full_channel_error.h ccan/ccan/cdump/tools/cdump-enumstr
 	ccan/ccan/cdump/tools/cdump-enumstr channeld/full_channel_error.h > $@
 
-lightningd/lightning_channeld: $(CHANNELD_OBJS) $(CHANNELD_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(HSMD_CLIENT_OBJS)
+lightningd/lightning_channeld: $(CHANNELD_OBJS) $(WIRE_ONION_OBJS) $(CHANNELD_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(HSMD_CLIENT_OBJS)
 
 include channeld/test/Makefile

channeld/channeld_htlc.h

@@ -27,7 +27,7 @@ struct htlc {
 	const u8 *routing;
 
 	/* Blinding (optional). */
-	struct pubkey *path_key;
+	struct pubkey *blinding;
 
 	/* Should we immediately fail this htlc? */
 	bool fail_immediate;
@@ -56,8 +56,7 @@ static inline size_t htlc_hash(u64 id)
 {
 	return siphash24(siphash_seed(), &id, sizeof(id));
 }
-HTABLE_DEFINE_DUPS_TYPE(struct htlc, htlc_key, htlc_hash, htlc_cmp,
-			htlc_map);
+HTABLE_DEFINE_TYPE(struct htlc, htlc_key, htlc_hash, htlc_cmp, htlc_map);
 
 static inline struct htlc *htlc_get(struct htlc_map *htlcs, u64 id, enum side owner)
 {

channeld/channeld_wire.csv

@@ -10,14 +10,11 @@
 #include <common/derive_basepoints.h>
 #include <common/features.h>
 #include <common/fee_states.h>
-#include <wire/peer_wire.h>
 
 # Begin!  (passes gossipd-client fd)
 msgtype,channeld_init,1000
 msgdata,channeld_init,chainparams,chainparams,
 msgdata,channeld_init,our_features,feature_set,
-msgdata,channeld_init,num_hsm_capabilities,u16,
-msgdata,channeld_init,hsm_capabilities,u32,num_hsm_capabilities
 msgdata,channeld_init,channel_id,channel_id,
 msgdata,channeld_init,funding,bitcoin_outpoint,
 msgdata,channeld_init,funding_satoshi,amount_sat,
@@ -37,10 +34,17 @@ msgdata,channeld_init,remote_basepoints,basepoints,
 msgdata,channeld_init,remote_per_commit,pubkey,
 msgdata,channeld_init,old_remote_per_commit,pubkey,
 msgdata,channeld_init,opener,enum side,
+msgdata,channeld_init,fee_base,u32,
+msgdata,channeld_init,fee_proportional,u32,
+msgdata,channeld_init,htlc_minimum_msat,amount_msat,
+msgdata,channeld_init,htlc_maximum_msat,amount_msat,
 msgdata,channeld_init,local_msatoshi,amount_msat,
 msgdata,channeld_init,our_basepoints,basepoints,
 msgdata,channeld_init,our_funding_pubkey,pubkey,
+msgdata,channeld_init,local_node_id,node_id,
+msgdata,channeld_init,remote_node_id,node_id,
 msgdata,channeld_init,commit_msec,u32,
+msgdata,channeld_init,cltv_delta,u16,
 msgdata,channeld_init,last_was_revoke,bool,
 msgdata,channeld_init,num_last_sent_commit,u16,
 msgdata,channeld_init,last_sent_commit,changed_htlc,num_last_sent_commit
@@ -63,27 +67,32 @@ msgdata,channeld_init,final_scriptpubkey,u8,final_scriptpubkey_len
 msgdata,channeld_init,flags,u8,
 msgdata,channeld_init,init_peer_pkt_len,u16,
 msgdata,channeld_init,init_peer_pkt,u8,init_peer_pkt_len
+msgdata,channeld_init,reached_announce_depth,bool,
 msgdata,channeld_init,last_remote_secret,secret,
 msgdata,channeld_init,flen,u16,
 msgdata,channeld_init,their_features,u8,flen
 msgdata,channeld_init,upfront_shutdown_script_len,u16,
 msgdata,channeld_init,upfront_shutdown_script,u8,upfront_shutdown_script_len
+msgdata,channeld_init,remote_ann_node_sig,?secp256k1_ecdsa_signature,
+msgdata,channeld_init,remote_ann_bitcoin_sig,?secp256k1_ecdsa_signature,
 msgdata,channeld_init,desired_type,channel_type,
+msgdata,channeld_init,dev_fast_gossip,bool,
 msgdata,channeld_init,dev_disable_commit,?u32,
 msgdata,channeld_init,num_penalty_bases,u32,
 msgdata,channeld_init,pbases,penalty_base,num_penalty_bases
+msgdata,channeld_init,reestablish_only,bool,
+msgdata,channeld_init,channel_update_len,u16,
+msgdata,channeld_init,channel_update,u8,channel_update_len
 msgdata,channeld_init,experimental_upgrade,bool,
 msgdata,channeld_init,num_inflights,u16,
 msgdata,channeld_init,inflights,inflight,num_inflights
-msgdata,channeld_init,scid_alias,short_channel_id,
-
-# channeld->lightningd: successfully negotated reestablishment.
-msgtype,channeld_reestablished,1101
 
 # master->channeld funding hit new depth(funding locked if >= lock depth)
+# alias != NULL if zeroconf and short_channel_id == NULL
 # short_channel_id != NULL once we have 3+ confirmations
 msgtype,channeld_funding_depth,1002
 msgdata,channeld_funding_depth,short_channel_id,?short_channel_id,
+msgdata,channeld_funding_depth,alias_local,?short_channel_id,
 msgdata,channeld_funding_depth,depth,u32,
 msgdata,channeld_funding_depth,splicing,bool,
 msgdata,channeld_funding_depth,txid,bitcoin_txid,
@@ -94,7 +103,7 @@ msgdata,channeld_offer_htlc,amount_msat,amount_msat,
 msgdata,channeld_offer_htlc,cltv_expiry,u32,
 msgdata,channeld_offer_htlc,payment_hash,sha256,
 msgdata,channeld_offer_htlc,onion_routing_packet,u8,1366
-msgdata,channeld_offer_htlc,path_key,?pubkey,
+msgdata,channeld_offer_htlc,blinding,?pubkey,
 
 # Reply; synchronous since IDs have to increment.
 msgtype,channeld_offer_htlc_reply,1104
@@ -126,18 +135,6 @@ msgdata,channeld_got_splice_locked,locked_txid,bitcoin_txid,
 
 #include <common/penalty_base.h>
 
-subtype,local_anchor_info
-subtypedata,local_anchor_info,commitment_weight,u32,
-subtypedata,local_anchor_info,commitment_fee,amount_sat,
-subtypedata,local_anchor_info,anchor_point,bitcoin_outpoint,
-
-# lightningd needs to track our anchor outputs on remote txs.
-# This includes splices, so there could be more than one!
-msgtype,channeld_local_anchor_info,1003
-msgdata,channeld_local_anchor_info,remote_commitnum,u64,
-msgdata,channeld_local_anchor_info,num_anchors,u16,
-msgdata,channeld_local_anchor_info,anchors,local_anchor_info,num_anchors
-
 # When we send a commitment_signed message, tell master.
 msgtype,channeld_sending_commitsig,1020
 msgdata,channeld_sending_commitsig,commitnum,u64,
@@ -147,7 +144,9 @@ msgdata,channeld_sending_commitsig,blockheight_states,height_states,
 # SENT_ADD_COMMIT, SENT_REMOVE_ACK_COMMIT, SENT_ADD_ACK_COMMIT, SENT_REMOVE_COMMIT
 msgdata,channeld_sending_commitsig,num_changed,u16,
 msgdata,channeld_sending_commitsig,changed,changed_htlc,num_changed
-
+msgdata,channeld_sending_commitsig,commit_sig,bitcoin_signature,
+msgdata,channeld_sending_commitsig,num_htlc_sigs,u16,
+msgdata,channeld_sending_commitsig,htlc_sigs,bitcoin_signature,num_htlc_sigs
 
 # Wait for reply, to make sure it's on disk before we send commit.
 msgtype,channeld_sending_commitsig_reply,1120
@@ -209,7 +208,6 @@ msgdata,channeld_splice_init,psbt,wally_psbt,
 msgdata,channeld_splice_init,relative_amount,s64,
 msgdata,channeld_splice_init,feerate_per_kw,u32,
 msgdata,channeld_splice_init,force_feerate,bool,
-msgdata,channeld_splice_init,skip_stfu,bool,
 
 # channeld->master: hello, I started a channel splice open
 msgtype,channeld_splice_confirmed_init,7205
@@ -223,7 +221,6 @@ msgdata,channeld_splice_update,psbt,wally_psbt,
 msgtype,channeld_splice_confirmed_update,7207
 msgdata,channeld_splice_confirmed_update,psbt,wally_psbt,
 msgdata,channeld_splice_confirmed_update,commitments_secured,bool,
-msgdata,channeld_splice_confirmed_update,signatures_secured,bool,
 
 # channeld->master: Lookup a transaction
 msgtype,channeld_splice_lookup_tx,7208
@@ -243,10 +240,6 @@ msgtype,channeld_splice_confirmed_signed,7213
 msgdata,channeld_splice_confirmed_signed,tx,bitcoin_tx,
 msgdata,channeld_splice_confirmed_signed,output_index,u32,
 
-# channeld->master: Splice signatures are about to be sent
-msgtype,channeld_splice_sending_sigs,7214
-msgdata,channeld_splice_sending_sigs,tx,bitcoin_txid,
-
 # channeld->master: A feerate error has occured
 msgtype,channeld_splice_feerate_error,7215
 msgdata,channeld_splice_feerate_error,fee,amount_msat,
@@ -254,7 +247,6 @@ msgdata,channeld_splice_feerate_error,too_high,bool,
 
 # channeld->master: Add an inflight to the DB
 msgtype,channeld_add_inflight,7216
-msgdata,channeld_add_inflight,remote_funding,pubkey,
 msgdata,channeld_add_inflight,tx_id,bitcoin_txid,
 msgdata,channeld_add_inflight,tx_outnum,u32,
 msgdata,channeld_add_inflight,feerate,u32,
@@ -262,7 +254,6 @@ msgdata,channeld_add_inflight,satoshis,amount_sat,
 msgdata,channeld_add_inflight,splice_amount,s64,
 msgdata,channeld_add_inflight,psbt,wally_psbt,
 msgdata,channeld_add_inflight,i_am_initiator,bool,
-msgdata,channeld_add_inflight,force_sign_first,bool,
 
 # master->channeld: Inflight saved successfully
 msgtype,channeld_got_inflight,7217
@@ -272,7 +263,6 @@ msgtype,channeld_update_inflight,7219
 msgdata,channeld_update_inflight,psbt,wally_psbt,
 msgdata,channeld_update_inflight,last_tx,?bitcoin_tx,
 msgdata,channeld_update_inflight,last_sig,?bitcoin_signature,
-msgdata,channeld_update_inflight,locked_scid,?short_channel_id,
 
 # channeld->master: A funding error has occured
 msgtype,channeld_splice_funding_error,7220
@@ -284,22 +274,6 @@ msgdata,channeld_splice_funding_error,opener_error,bool,
 msgtype,channeld_splice_state_error,7221
 msgdata,channeld_splice_state_error,state_error,wirestring,
 
-# channeld->master: Peer rejected our splice
-msgtype,channeld_splice_abort,7223
-msgdata,channeld_splice_abort,did_i_initiate,bool,
-msgdata,channeld_splice_abort,inflight_outpoint,?bitcoin_outpoint,
-msgdata,channeld_splice_abort,reason,wirestring,
-
-# master->channeld: Please enter stfu mode
-msgtype,channeld_stfu,7224
-
-# channeld->master: Entered stfu result
-msgtype,channeld_confirmed_stfu,7225
-msgdata,channeld_confirmed_stfu,available_funds,amount_msat,
-
-# master->channeld: Please enter perform tx_abort
-msgtype,channeld_abort,7226
-
 # Tell peer to shut down channel.
 msgtype,channeld_send_shutdown,1023
 msgdata,channeld_send_shutdown,final_index,?u32,
@@ -335,10 +309,18 @@ msgdata,channeld_dev_memleak_reply,leak,bool,
 
 # Peer presented proof it was from the future.
 msgtype,channeld_fail_fallen_behind,1028
+# This is NULL if option_static_remotekey.
+msgdata,channeld_fail_fallen_behind,remote_per_commitment_point,?pubkey,
+
+# Handle a channel-specific configuration change
+msgtype,channeld_config_channel,1029
+msgdata,channeld_config_channel,feerate_base,?u32,
+msgdata,channeld_config_channel,feerate_ppm,?u32,
+msgdata,channeld_config_channel,htlc_minimum,?amount_msat,
+msgdata,channeld_config_channel,htlc_maximum,?amount_msat,
 
 # When we receive announcement_signatures for channel announce
 msgtype,channeld_got_announcement,1017
-msgdata,channeld_got_announcement,scid,short_channel_id,
 msgdata,channeld_got_announcement,remote_ann_node_sig,secp256k1_ecdsa_signature,
 msgdata,channeld_got_announcement,remote_ann_bitcoin_sig,secp256k1_ecdsa_signature,
 
@@ -349,6 +331,36 @@ msgdata,channeld_send_error,reason,wirestring,
 # Tell master channeld has sent the error message.
 msgtype,channeld_send_error_reply,1108
 
+# Tell channeld about the latest channel_update
+msgtype,channeld_channel_update,1001
+msgdata,channeld_channel_update,len,u16,
+msgdata,channeld_channel_update,msg,u8,len
+
+# Tell lightningd we used the latest channel_update for an error.
+msgtype,channeld_used_channel_update,1102
+
+# Channeld: tell gossipd to make this channel_update.
+msgtype,channeld_local_channel_update,1013
+msgdata,channeld_local_channel_update,short_channel_id,short_channel_id,
+msgdata,channeld_local_channel_update,disable,bool,
+msgdata,channeld_local_channel_update,cltv_expiry_delta,u16,
+msgdata,channeld_local_channel_update,htlc_minimum_msat,amount_msat,
+msgdata,channeld_local_channel_update,fee_base_msat,u32,
+msgdata,channeld_local_channel_update,fee_proportional_millionths,u32,
+msgdata,channeld_local_channel_update,htlc_maximum_msat,amount_msat,
+msgdata,channeld_local_channel_update,public,bool,
+
+# Channeld: tell gossipd about our channel_announcement
+msgtype,channeld_local_channel_announcement,1014
+msgdata,channeld_local_channel_announcement,len,u16,
+msgdata,channeld_local_channel_announcement,cannounce,u8,len
+
+# Channeld: tell gossipd about this (as-yet) unannounced channel
+msgtype,channeld_local_private_channel,1015
+msgdata,channeld_local_private_channel,capacity,amount_sat,
+msgdata,channeld_local_private_channel,len,u16,
+msgdata,channeld_local_private_channel,features,u8,len
+
 # Ask channeld to quiesce.
 msgtype,channeld_dev_quiesce,1009
 msgtype,channeld_dev_quiesce_reply,1109
@@ -360,7 +372,3 @@ msgdata,channeld_upgraded,new_type,channel_type,
 # Tell peer about our latest and greatest blockheight.
 msgtype,channeld_blockheight,1012
 msgdata,channeld_blockheight,blockheight,u32,
-
-# Tell channeld about peer's shachain seed.
-msgtype,channeld_dev_peer_shachain,1013
-msgdata,channeld_dev_peer_shachain,seed,sha256,

channeld/commit_tx.c

@@ -53,7 +53,7 @@ bool commit_tx_amount_trimmed(const struct htlc **htlcs,
 		if (trim(htlcs[i], feerate_per_kw, dust_limit,
 			 option_anchor_outputs, option_anchors_zero_fee_htlc_tx,
 			 side)) {
-			if (!amount_msat_accumulate(amt, htlcs[i]->amount))
+			if (!amount_msat_add(amt, *amt, htlcs[i]->amount))
 				return false;
 		}
 	}
@@ -128,8 +128,7 @@ struct bitcoin_tx *commit_tx(const tal_t *ctx,
 			     u64 obscured_commitment_number,
 			     bool option_anchor_outputs,
 			     bool option_anchors_zero_fee_htlc_tx,
-			     enum side side,
-			     int *anchor_outnum)
+			     enum side side)
 {
 	struct amount_sat base_fee;
 	struct amount_msat total_pay;
@@ -140,8 +139,7 @@ struct bitcoin_tx *commit_tx(const tal_t *ctx,
 	u32 *cltvs;
 	bool to_local, to_remote;
 	struct htlc *dummy_to_local = (struct htlc *)0x01,
-		*dummy_to_remote = (struct htlc *)0x02,
-		*dummy_other_anchor = (struct htlc *)0x03;
+		*dummy_to_remote = (struct htlc *)0x02;
 	const u8 *funding_wscript = bitcoin_redeem_2of2(tmpctx,
 							local_funding_key,
 							remote_funding_key);
@@ -356,7 +354,8 @@ struct bitcoin_tx *commit_tx(const tal_t *ctx,
 		 * However, valgrind will warn us something wierd is happening */
 		SUPERVERBOSE("# to_remote amount %"PRIu64" P2WPKH(%s)\n",
 			     amount.satoshis, /* Raw: BOLT 3 output match */
-			     fmt_pubkey(tmpctx, &keyset->other_payment_key));
+			     type_to_string(tmpctx, struct pubkey,
+					    &keyset->other_payment_key));
 		n++;
 
 		to_remote = true;
@@ -380,11 +379,9 @@ struct bitcoin_tx *commit_tx(const tal_t *ctx,
 			n++;
 		}
 
-		/* With anchors, the caller really wants to know what
-		 * is the LOCAL anchor for the REMOTE side. */
 		if (to_remote || untrimmed != 0) {
 			tx_add_anchor_output(tx, remote_funding_key);
-			(*htlcmap)[n] = dummy_other_anchor;
+			(*htlcmap)[n] = NULL;
 			n++;
 		}
 	}
@@ -436,21 +433,17 @@ struct bitcoin_tx *commit_tx(const tal_t *ctx,
 	bitcoin_tx_add_input(tx, funding,
 			     sequence, NULL, funding_sats, NULL, funding_wscript);
 
-	/* Identify the direct outputs (to_us, to_them), and the local anchor */
-	if (direct_outputs != NULL)
+	/* Identify the direct outputs (to_us, to_them). */
+	if (direct_outputs != NULL) {
 		direct_outputs[LOCAL] = direct_outputs[REMOTE] = NULL;
-
-	*anchor_outnum = -1;
-	for (size_t i = 0; i < tx->wtx->num_outputs; i++) {
-		if ((*htlcmap)[i] == dummy_to_local) {
-			(*htlcmap)[i] = NULL;
-			direct_outputs[LOCAL] = tx->wtx->outputs + i;
-		} else if ((*htlcmap)[i] == dummy_to_remote) {
-			(*htlcmap)[i] = NULL;
-			direct_outputs[REMOTE] = tx->wtx->outputs + i;
-		} else if ((*htlcmap)[i] == dummy_other_anchor) {
-			(*htlcmap)[i] = NULL;
-			*anchor_outnum = i;
+		for (size_t i = 0; i < tx->wtx->num_outputs; i++) {
+			if ((*htlcmap)[i] == dummy_to_local) {
+				(*htlcmap)[i] = NULL;
+				direct_outputs[LOCAL] = tx->wtx->outputs + i;
+			} else if ((*htlcmap)[i] == dummy_to_remote) {
+				(*htlcmap)[i] = NULL;
+				direct_outputs[REMOTE] = tx->wtx->outputs + i;
+			}
 		}
 	}
 

channeld/commit_tx.h

@@ -63,9 +63,8 @@ bool commit_tx_amount_trimmed(const struct htlc **htlcs,
  * @obscured_commitment_number: number to encode in commitment transaction
  * @direct_outputs: If non-NULL, fill with pointers to the direct (non-HTLC) outputs (or NULL if none).
  * @option_anchor_outputs: does option_anchor_outputs apply to this channel?
- * @option_anchors_zero_fee_htlc_tx: does option_anchors_zero_fee_htlc_tx apply to this channel?
  * @side: side to generate commitment transaction for.
- * @anchor_outnum: set to index of local anchor, or -1 if none.
+ * @option_anchor_outputs: does option_anchor_outputs apply to this channel?
  *
  * We need to be able to generate the remote side's tx to create signatures,
  * but the BOLT is expressed in terms of generating our local commitment
@@ -91,7 +90,6 @@ struct bitcoin_tx *commit_tx(const tal_t *ctx,
 			     u64 obscured_commitment_number,
 			     bool option_anchor_outputs,
 			     bool option_anchors_zero_fee_htlc_tx,
-			     enum side side,
-			     int *anchor_outnum);
+			     enum side side);
 
 #endif /* LIGHTNING_CHANNELD_COMMIT_TX_H */

channeld/full_channel.c

@@ -13,15 +13,18 @@
 #include <common/keyset.h>
 #include <common/memleak.h>
 #include <common/status.h>
+#include <common/type_to_string.h>
 #include <stdio.h>
   /* Needs to be at end, since it doesn't include its own hdrs */
   #include "full_channel_error_names_gen.h"
 
+#if DEVELOPER
 static void memleak_help_htlcmap(struct htable *memtable,
 				 struct htlc_map *htlcs)
 {
 	memleak_scan_htable(memtable, &htlcs->raw);
 }
+#endif /* DEVELOPER */
 
 /* This is a dangerous thing!  Because we apply HTLCs in many places
  * in bulk, we can temporarily go negative.  You must check balance_ok()
@@ -226,7 +229,7 @@ static bool sum_offered_msatoshis(struct amount_msat *total,
 	*total = AMOUNT_MSAT(0);
 	for (i = 0; i < tal_count(htlcs); i++) {
 		if (htlc_owner(htlcs[i]) == side) {
-			if (!amount_msat_accumulate(total, htlcs[i]->amount))
+			if (!amount_msat_add(total, *total, htlcs[i]->amount))
 				return false;
 		}
 	}
@@ -241,7 +244,7 @@ static void add_htlcs(struct bitcoin_tx ***txs,
 {
 	struct bitcoin_outpoint outpoint;
 	u32 htlc_feerate_per_kw;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 
 	if (channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX))
@@ -298,28 +301,38 @@ static void add_htlcs(struct bitcoin_tx ***txs,
 
 /* FIXME: We could cache these. */
 struct bitcoin_tx **channel_txs(const tal_t *ctx,
-				const struct bitcoin_outpoint *funding,
-				struct amount_sat funding_sats,
 				const struct htlc ***htlcmap,
 				struct wally_tx_output *direct_outputs[NUM_SIDES],
 				const u8 **funding_wscript,
 				const struct channel *channel,
 				const struct pubkey *per_commitment_point,
 				u64 commitment_number,
-				enum side side,
-				s64 splice_amnt,
-				s64 remote_splice_amnt,
-				int *other_anchor_outnum,
-				const struct pubkey funding_pubkeys[NUM_SIDES])
+				enum side side)
+{
+	return channel_splice_txs(ctx, &channel->funding, channel->funding_sats,
+				  htlcmap, direct_outputs, funding_wscript,
+				  channel, per_commitment_point,
+				  commitment_number, side, 0, 0);
+}
+
+struct bitcoin_tx **channel_splice_txs(const tal_t *ctx,
+				       const struct bitcoin_outpoint *funding,
+				       struct amount_sat funding_sats,
+				       const struct htlc ***htlcmap,
+				       struct wally_tx_output *direct_outputs[NUM_SIDES],
+				       const u8 **funding_wscript,
+				       const struct channel *channel,
+				       const struct pubkey *per_commitment_point,
+				       u64 commitment_number,
+				       enum side side,
+				       s64 splice_amnt,
+				       s64 remote_splice_amnt)
 {
 	struct bitcoin_tx **txs;
 	const struct htlc **committed;
 	struct keyset keyset;
 	struct amount_msat side_pay, other_side_pay;
 
-	if (!funding_pubkeys)
-		funding_pubkeys = channel->funding_pubkey;
-
 	if (!derive_keyset(per_commitment_point,
 			   &channel->basepoints[side],
 			   &channel->basepoints[!side],
@@ -333,8 +346,8 @@ struct bitcoin_tx **channel_txs(const tal_t *ctx,
 	/* Generating and saving witness script required to spend
 	 * the funding output */
 	*funding_wscript = bitcoin_redeem_2of2(ctx,
-					      &funding_pubkeys[side],
-					      &funding_pubkeys[!side]);
+					      &channel->funding_pubkey[side],
+					      &channel->funding_pubkey[!side]);
 
 	side_pay = channel->view[side].owed[side];
 	other_side_pay = channel->view[side].owed[!side];
@@ -355,8 +368,8 @@ struct bitcoin_tx **channel_txs(const tal_t *ctx,
 	txs[0] = commit_tx(
 	    txs, funding,
 	    funding_sats,
-	    &funding_pubkeys[side],
-	    &funding_pubkeys[!side],
+	    &channel->funding_pubkey[side],
+	    &channel->funding_pubkey[!side],
 	    channel->opener,
 	    channel->config[!side].to_self_delay,
 	    channel->lease_expiry,
@@ -365,15 +378,15 @@ struct bitcoin_tx **channel_txs(const tal_t *ctx,
 	    channel->config[side].dust_limit, side_pay,
 	    other_side_pay, committed, htlcmap, direct_outputs,
 	    commitment_number ^ channel->commitment_number_obscurer,
-	    channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED),
+	    channel_has(channel, OPT_ANCHOR_OUTPUTS),
 	    channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX),
-	    side, other_anchor_outnum);
+	    side);
 
 	/* Set the remote/local pubkeys on the commitment tx psbt */
 	psbt_input_add_pubkey(txs[0]->psbt, 0,
-			      &funding_pubkeys[side], false /* is_taproot */);
+			      &channel->funding_pubkey[side], false /* is_taproot */);
 	psbt_input_add_pubkey(txs[0]->psbt, 0,
-			      &funding_pubkeys[!side], false /* is_taproot */);
+			      &channel->funding_pubkey[!side], false /* is_taproot */);
 
 	add_htlcs(&txs, *htlcmap, channel, &keyset, side);
 
@@ -427,8 +440,10 @@ static bool get_room_above_reserve(const struct channel *channel,
 		status_debug("%s cannot afford htlc: would make balance %s"
 			     " below reserve %s",
 			     side_to_str(side),
-			     fmt_amount_msat(tmpctx, *remainder),
-			     fmt_amount_sat(tmpctx, reserve));
+			     type_to_string(tmpctx, struct amount_msat,
+					    remainder),
+			     type_to_string(tmpctx, struct amount_sat,
+					    &reserve));
 		return false;
 	}
 	return true;
@@ -466,7 +481,7 @@ static struct amount_sat fee_for_htlcs(const struct channel *channel,
 	u32 feerate = channel_feerate(channel, side);
 	struct amount_sat dust_limit = channel->config[side].dust_limit;
 	size_t untrimmed;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 
 	untrimmed = num_untrimmed_htlcs(side, dust_limit, feerate,
@@ -488,7 +503,7 @@ static bool htlc_dust(const struct channel *channel,
 		      struct amount_msat *trim_total)
 {
 	struct amount_sat dust_limit = channel->config[side].dust_limit;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 	struct amount_msat trim_rmvd = AMOUNT_MSAT(0);
 
@@ -526,7 +541,7 @@ static bool htlc_dust(const struct channel *channel,
  *
  * To mostly avoid this situation, at least from our side, we apply an
  * additional constraint when we're opener trying to add an HTLC: make
- * sure we can afford one more HTLC, even if fees increase.
+ * sure we can afford one more HTLC, even if fees increase by 100%.
  *
  * We could do this for the peer, as well, by rejecting their HTLC
  * immediately in this case.  But rejecting a remote HTLC here causes
@@ -548,7 +563,7 @@ static bool local_opener_has_fee_headroom(const struct channel *channel,
 	u32 feerate = channel_feerate(channel, LOCAL);
 	size_t untrimmed;
 	struct amount_sat fee;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 
 	assert(channel->opener == LOCAL);
@@ -563,17 +578,17 @@ static bool local_opener_has_fee_headroom(const struct channel *channel,
 
 	/* Now, how much would it cost us if feerate increases 100% and we added
 	 * another HTLC? */
-	fee = commit_tx_base_fee(marginal_feerate(feerate), untrimmed + 1,
+	fee = commit_tx_base_fee(2 * feerate, untrimmed + 1,
 				 option_anchor_outputs,
 				 option_anchors_zero_fee_htlc_tx);
 	if (amount_msat_greater_eq_sat(remainder, fee))
 		return true;
 
 	status_debug("Adding HTLC would leave us only %s: we need %s for"
-		     " another HTLC if fees increase from %uperkw to %uperkw",
-		     fmt_amount_msat(tmpctx, remainder),
-		     fmt_amount_sat(tmpctx, fee),
-		     feerate, marginal_feerate(feerate));
+		     " another HTLC if fees increase by 100%% to %uperkw",
+		     type_to_string(tmpctx, struct amount_msat, &remainder),
+		     type_to_string(tmpctx, struct amount_sat, &fee),
+		     feerate + feerate);
 	return false;
 }
 
@@ -584,7 +599,7 @@ static enum channel_add_err add_htlc(struct channel *channel,
 				     u32 cltv_expiry,
 				     const struct sha256 *payment_hash,
 				     const u8 routing[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)],
-				     const struct pubkey *path_key TAKES,
+				     const struct pubkey *blinding TAKES,
 				     struct htlc **htlcp,
 				     bool enforce_aggregate_limits,
 				     struct amount_sat *htlc_fee,
@@ -597,7 +612,7 @@ static enum channel_add_err add_htlc(struct channel *channel,
 	const struct htlc **committed, **adding, **removing;
 	const struct channel_view *view;
 	size_t htlc_count;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 	u32 feerate, feerate_ceil;
 
@@ -609,7 +624,7 @@ static enum channel_add_err add_htlc(struct channel *channel,
 	htlc->fail_immediate = false;
 
 	htlc->rhash = *payment_hash;
-	htlc->path_key = tal_dup_or_null(htlc, struct pubkey, path_key);
+	htlc->blinding = tal_dup_or_null(htlc, struct pubkey, blinding);
 	htlc->failed = NULL;
 	htlc->r = NULL;
 	htlc->routing = tal_dup_arr(htlc, u8, routing, TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE), 0);
@@ -651,7 +666,7 @@ static enum channel_add_err add_htlc(struct channel *channel,
 	 *        - SHOULD send a `warning` and close the connection, or send an
 	 *        `error` and fail the channel.
 	 */
-	if (amount_msat_is_zero(htlc->amount)) {
+	if (amount_msat_eq(htlc->amount, AMOUNT_MSAT(0))) {
 		return CHANNEL_ERR_HTLC_BELOW_MINIMUM;
 	}
 	if (amount_msat_less(htlc->amount, channel->config[recipient].htlc_minimum)) {
@@ -776,8 +791,10 @@ static enum channel_add_err add_htlc(struct channel *channel,
 		if (channel->opener== sender) {
 			if (amount_msat_less_sat(remainder, fee)) {
 				status_debug("Cannot afford fee %s with %s above reserve",
-					     fmt_amount_sat(tmpctx, fee),
-					     fmt_amount_msat(tmpctx, remainder));
+					     type_to_string(tmpctx, struct amount_sat,
+							    &fee),
+					     type_to_string(tmpctx, struct amount_msat,
+							    &remainder));
 				return CHANNEL_ERR_CHANNEL_CAPACITY_EXCEEDED;
 			}
 
@@ -818,8 +835,12 @@ static enum channel_add_err add_htlc(struct channel *channel,
 				*htlc_fee = fee;
 			if (amount_msat_less_sat(remainder, fee)) {
 				status_debug("Funder could not afford own fee %s with %s above reserve",
-					     fmt_amount_sat(tmpctx, fee),
-					     fmt_amount_msat(tmpctx, remainder));
+					     type_to_string(tmpctx,
+							    struct amount_sat,
+							    &fee),
+					     type_to_string(tmpctx,
+							    struct amount_msat,
+							    &remainder));
 				return CHANNEL_ERR_CHANNEL_CAPACITY_EXCEEDED;
 			}
 			fee = fee_for_htlcs(channel,
@@ -831,9 +852,13 @@ static enum channel_add_err add_htlc(struct channel *channel,
 			if (htlc_fee && amount_sat_greater(fee, *htlc_fee))
 				*htlc_fee = fee;
 			if (amount_msat_less_sat(remainder, fee)) {
-				status_debug("Funder could not afford peer's fee `%s with %s above reserve",
-					     fmt_amount_sat(tmpctx, fee),
-					     fmt_amount_msat(tmpctx, remainder));
+				status_debug("Funder could not afford peer's fee %s with %s above reserve",
+					     type_to_string(tmpctx,
+							    struct amount_sat,
+							    &fee),
+					     type_to_string(tmpctx,
+							    struct amount_msat,
+							    &remainder));
 				return CHANNEL_ERR_CHANNEL_CAPACITY_EXCEEDED;
 			}
 		}
@@ -902,7 +927,7 @@ enum channel_add_err channel_add_htlc(struct channel *channel,
 				      u32 cltv_expiry,
 				      const struct sha256 *payment_hash,
 				      const u8 routing[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)],
-				      const struct pubkey *path_key TAKES,
+				      const struct pubkey *blinding TAKES,
 				      struct htlc **htlcp,
 				      struct amount_sat *htlc_fee,
 				      bool err_immediate_failures)
@@ -922,7 +947,7 @@ enum channel_add_err channel_add_htlc(struct channel *channel,
 		status_broken("Peer sent out-of-order HTLC ids (is that you, old c-lightning node?)");
 
 	return add_htlc(channel, state, id, amount, cltv_expiry,
-			payment_hash, routing, path_key,
+			payment_hash, routing, blinding,
 			htlcp, true, htlc_fee, err_immediate_failures);
 }
 
@@ -1183,8 +1208,8 @@ static int change_htlcs(struct channel *channel,
 				      "%s: %s balance underflow: %s -> %"PRId64,
 				      side_to_str(sidechanged),
 				      side_to_str(i),
-				      fmt_amount_msat(tmpctx,
-						      channel->view[sidechanged].owed[i]),
+				      type_to_string(tmpctx, struct amount_msat,
+						     &channel->view[sidechanged].owed[i]),
 				      owed[i].msat);
 		}
 	}
@@ -1218,10 +1243,9 @@ u32 approx_max_feerate(const struct channel *channel)
 {
 	size_t num;
 	u64 weight;
-	struct amount_msat avail;
-	struct balance balance;
+	struct amount_sat avail;
 	const struct htlc **committed, **adding, **removing;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
 
 	gather_htlcs(tmpctx, channel, !channel->opener,
@@ -1233,18 +1257,7 @@ u32 approx_max_feerate(const struct channel *channel)
 	weight = commit_tx_base_weight(num, option_anchor_outputs, option_anchors_zero_fee_htlc_tx);
 
 	/* Available is their view */
-	to_balance(&balance, channel->view[!channel->opener].owed[channel->opener]);
-
-	/* But take into account any pending added htlcs (conservatively, we ignore removed ones) */
-	for (size_t i = 0; i < tal_count(adding); i++)
-		balance_add_htlc(&balance, adding[i], channel->opener);
-
-	if (!balance_ok(&balance, &avail)) {
-		status_broken("Negative balance %"PRId64" after removing %zu htlcs!",
-			      balance.msat,
-			      tal_count(removing));
-		return 0;
-	}
+	avail = amount_msat_to_sat_round_down(channel->view[!channel->opener].owed[channel->opener]);
 
 	/* BOLT #3:
 	 * If `option_anchors` applies to the commitment
@@ -1253,16 +1266,16 @@ u32 approx_max_feerate(const struct channel *channel)
 	 * `to_remote`).
 	 */
 	if ((option_anchor_outputs || option_anchors_zero_fee_htlc_tx)
-	    && !amount_msat_sub_sat(&avail, avail, AMOUNT_SAT(660))) {
-		avail = AMOUNT_MSAT(0);
+	    && !amount_sat_sub(&avail, avail, AMOUNT_SAT(660))) {
+		avail = AMOUNT_SAT(0);
 	} else {
 		/* We should never go below reserve. */
-		if (!amount_msat_sub_sat(&avail, avail,
-					 channel->config[!channel->opener].channel_reserve))
-		avail = AMOUNT_MSAT(0);
+		if (!amount_sat_sub(&avail, avail,
+				    channel->config[!channel->opener].channel_reserve))
+		avail = AMOUNT_SAT(0);
 	}
 
-	return avail.millisatoshis / weight; /* Raw: once-off reverse feerate*/
+	return avail.satoshis / weight * 1000; /* Raw: once-off reverse feerate*/
 }
 
 /* Is the sum of trimmed htlcs, as this new feerate, above our
@@ -1302,17 +1315,12 @@ bool can_opener_afford_feerate(const struct channel *channel, u32 feerate_per_kw
 	struct amount_sat dust_limit = channel->config[!channel->opener].dust_limit;
 	size_t untrimmed;
 	const struct htlc **committed, **adding, **removing;
-	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS_DEPRECATED);
+	bool option_anchor_outputs = channel_has(channel, OPT_ANCHOR_OUTPUTS);
 	bool option_anchors_zero_fee_htlc_tx = channel_has(channel, OPT_ANCHORS_ZERO_FEE_HTLC_TX);
-	struct balance balance;
-	struct amount_msat available;
 
 	gather_htlcs(tmpctx, channel, !channel->opener,
 		     &committed, &removing, &adding);
 
-	status_debug("Committed %zu, removing %zu, adding %zu",
-		     tal_count(committed), tal_count(removing), tal_count(adding));
-
 	untrimmed = commit_tx_num_untrimmed(committed, feerate_per_kw, dust_limit,
 					    option_anchor_outputs,
 					    option_anchors_zero_fee_htlc_tx,
@@ -1340,7 +1348,8 @@ bool can_opener_afford_feerate(const struct channel *channel, u32 feerate_per_kw
 	    && !amount_sat_add(&fee, fee, AMOUNT_SAT(660)))
 		status_failed(STATUS_FAIL_INTERNAL_ERROR,
 			      "Cannot add 660 sats to %s for anchor",
-			      fmt_amount_sat(tmpctx, fee));
+			      type_to_string(tmpctx, struct amount_sat,
+					     &fee));
 
 	/* BOLT #2:
 	 *
@@ -1356,34 +1365,20 @@ bool can_opener_afford_feerate(const struct channel *channel, u32 feerate_per_kw
 			    channel->config[!channel->opener].channel_reserve))
 		status_failed(STATUS_FAIL_INTERNAL_ERROR,
 			      "Cannot add fee %s and reserve %s",
-			      fmt_amount_sat(tmpctx, fee),
-			      fmt_amount_sat(tmpctx,
-					     channel->config[!channel->opener].channel_reserve));
+			      type_to_string(tmpctx, struct amount_sat,
+					     &fee),
+			      type_to_string(tmpctx, struct amount_sat,
+					     &channel->config[!channel->opener].channel_reserve));
 
-	/* The amount we have needs to take into account that we're
-	 * adding and removing htlcs */
-	to_balance(&balance, channel->view[!channel->opener].owed[channel->opener]);
-	for (size_t i = 0; i < tal_count(removing); i++)
-		balance_remove_htlc(&balance, removing[i], channel->opener);
-	for (size_t i = 0; i < tal_count(adding); i++)
-		balance_add_htlc(&balance, adding[i], channel->opener);
-
-	if (!balance_ok(&balance, &available)) {
-		status_broken("Negative balance %"PRId64" after removing %zu htlcs!",
-			      balance.msat,
-			      tal_count(removing));
-		return false;
-	}
-
-	status_debug("We need %s at feerate %u for %zu untrimmed htlcs: we have %s/%s (will have %s)",
-		     fmt_amount_sat(tmpctx, needed),
+	status_debug("We need %s at feerate %u for %zu untrimmed htlcs: we have %s/%s",
+		     type_to_string(tmpctx, struct amount_sat, &needed),
 		     feerate_per_kw, untrimmed,
-		     fmt_amount_msat(tmpctx,
-				     channel->view[LOCAL].owed[channel->opener]),
-		     fmt_amount_msat(tmpctx,
-				     channel->view[REMOTE].owed[channel->opener]),
-		     fmt_amount_msat(tmpctx, available));
-	return amount_msat_greater_eq_sat(available, needed);
+		     type_to_string(tmpctx, struct amount_msat,
+				    &channel->view[LOCAL].owed[channel->opener]),
+		     type_to_string(tmpctx, struct amount_msat,
+				    &channel->view[REMOTE].owed[channel->opener]));
+	return amount_msat_greater_eq_sat(channel->view[!channel->opener].owed[channel->opener],
+					  needed);
 }
 
 bool channel_update_feerate(struct channel *channel, u32 feerate_per_kw)
@@ -1609,9 +1604,11 @@ bool channel_force_htlcs(struct channel *channel,
 			     " payment_hash=%s %s",
 			     i, tal_count(htlcs),
 			     htlcs[i]->id,
-			     fmt_amount_msat(tmpctx, htlcs[i]->amount),
+			     type_to_string(tmpctx, struct amount_msat,
+					    &htlcs[i]->amount),
 			     htlcs[i]->cltv_expiry,
-			     fmt_sha256(tmpctx, &htlcs[i]->payment_hash),
+			     type_to_string(tmpctx, struct sha256,
+					    &htlcs[i]->payment_hash),
 			     htlcs[i]->payment_preimage ? "(have preimage)"
 			     : htlcs[i]->failed ? "(failed)" : "");
 
@@ -1620,7 +1617,7 @@ bool channel_force_htlcs(struct channel *channel,
 			     htlcs[i]->cltv_expiry,
 			     &htlcs[i]->payment_hash,
 			     htlcs[i]->onion_routing_packet,
-			     htlcs[i]->path_key,
+			     htlcs[i]->blinding,
 			     &htlc, false, NULL, false);
 		if (e != CHANNEL_ERR_ADD_OK) {
 			status_broken("%s HTLC %"PRIu64" failed error %u",
@@ -1658,13 +1655,13 @@ bool channel_force_htlcs(struct channel *channel,
 
 const char *channel_add_err_name(enum channel_add_err e)
 {
-	static char invalidbuf[sizeof("UNKNOWN ") + STR_MAX_CHARS(e)];
+	static char invalidbuf[sizeof("INVALID ") + STR_MAX_CHARS(e)];
 
 	for (size_t i = 0; enum_channel_add_err_names[i].name; i++) {
 		if (enum_channel_add_err_names[i].v == e)
 			return enum_channel_add_err_names[i].name;
 	}
-	snprintf(invalidbuf, sizeof(invalidbuf), "UNKNOWN %i", e);
+	snprintf(invalidbuf, sizeof(invalidbuf), "INVALID %i", e);
 	return invalidbuf;
 }
 

channeld/full_channel.h

@@ -55,38 +55,42 @@ struct channel *new_full_channel(const tal_t *ctx,
 /**
  * channel_txs: Get the current commitment and htlc txs for the channel.
  * @ctx: tal context to allocate return value from.
- * @funding: the outpoint we're spending
- * @funding_sats: the amount of @funding
+ * @channel: The channel to evaluate
  * @htlc_map: Pointer to htlcs for each tx output (allocated off @ctx).
  * @direct_outputs: If non-NULL, fill with pointers to the direct (non-HTLC) outputs (or NULL if none).
  * @funding_wscript: Pointer to wscript for the funding tx output
- * @channel: The channel to evaluate
  * @per_commitment_point: Per-commitment point to determine keys
  * @commitment_number: The index of this commitment.
  * @side: which side to get the commitment transaction for
- * @local_splice_amnt: how much is being spliced in (or out, if -ve) of local side.
- * @remote_splice_amnt: how much is being spliced in (or out, if -ve) of remote side.
- * @other_anchor_outnum: which output (-1 if none) is the !!side anchor
- * @funding_pubkeys: The funding pubkeys (specify NULL to use channel's value).
  *
  * Returns the unsigned commitment transaction for the committed state
  * for @side, followed by the htlc transactions in output order and
  * fills in @htlc_map, or NULL on key derivation failure.
  */
 struct bitcoin_tx **channel_txs(const tal_t *ctx,
-				const struct bitcoin_outpoint *funding,
-				struct amount_sat funding_sats,
 				const struct htlc ***htlcmap,
 				struct wally_tx_output *direct_outputs[NUM_SIDES],
 				const u8 **funding_wscript,
 				const struct channel *channel,
 				const struct pubkey *per_commitment_point,
 				u64 commitment_number,
-				enum side side,
-				s64 local_splice_amnt,
-				s64 remote_splice_amnt,
-				int *local_anchor_outnum,
-				const struct pubkey funding_pubkeys[NUM_SIDES]);
+				enum side side);
+
+/* Version of `channel_txs` that lets you specify a custom funding outpoint
+ * and funding_sats.
+ */
+struct bitcoin_tx **channel_splice_txs(const tal_t *ctx,
+				       const struct bitcoin_outpoint *funding,
+				       struct amount_sat funding_sats,
+				       const struct htlc ***htlcmap,
+				       struct wally_tx_output *direct_outputs[NUM_SIDES],
+				       const u8 **funding_wscript,
+				       const struct channel *channel,
+				       const struct pubkey *per_commitment_point,
+				       u64 commitment_number,
+				       enum side side,
+				       s64 splice_amnt,
+				       s64 remote_splice_amnt);
 
 /**
  * actual_feerate: what is the actual feerate for the local side.

channeld/inflight.c

@@ -1,7 +1,6 @@
 #include "config.h"
 #include <assert.h>
 #include <bitcoin/psbt.h>
-#include <bitcoin/short_channel_id.h>
 #include <channeld/inflight.h>
 #include <wire/wire.h>
 
@@ -10,30 +9,12 @@ struct inflight *fromwire_inflight(const tal_t *ctx, const u8 **cursor, size_t *
 	struct inflight *inflight = tal(ctx, struct inflight);
 
 	fromwire_bitcoin_outpoint(cursor, max, &inflight->outpoint);
-	fromwire_pubkey(cursor, max, &inflight->remote_funding);
 	inflight->amnt = fromwire_amount_sat(cursor, max);
-	inflight->remote_tx_sigs = fromwire_bool(cursor, max);
 	inflight->psbt = fromwire_wally_psbt(inflight, cursor, max);
 	inflight->splice_amnt = fromwire_s64(cursor, max);
-	int has_tx = fromwire_u8(cursor, max);
-	if(has_tx) {
-		inflight->last_tx = fromwire_bitcoin_tx(inflight, cursor, max);
-		fromwire_bitcoin_signature(cursor, max, &inflight->last_sig);
-	}
-	else {
-		inflight->last_tx = NULL;
-		memset(&inflight->last_sig, 0, sizeof(inflight->last_sig));
-	}
+	inflight->last_tx = fromwire_bitcoin_tx(inflight, cursor, max);
+	fromwire_bitcoin_signature(cursor, max, &inflight->last_sig);
 	inflight->i_am_initiator = fromwire_bool(cursor, max);
-	inflight->force_sign_first = fromwire_bool(cursor, max);
-	int has_locked_scid = fromwire_u8(cursor, max);
-	if (has_locked_scid) {
-		inflight->locked_scid = tal(inflight, struct short_channel_id);
-		*inflight->locked_scid = fromwire_short_channel_id(cursor, max);
-	}
-	else {
-		inflight->locked_scid = NULL;
-	}
 
 	return inflight;
 }
@@ -41,19 +22,21 @@ struct inflight *fromwire_inflight(const tal_t *ctx, const u8 **cursor, size_t *
 void towire_inflight(u8 **pptr, const struct inflight *inflight)
 {
 	towire_bitcoin_outpoint(pptr, &inflight->outpoint);
-	towire_pubkey(pptr, &inflight->remote_funding);
 	towire_amount_sat(pptr, inflight->amnt);
-	towire_bool(pptr, inflight->remote_tx_sigs);
 	towire_wally_psbt(pptr, inflight->psbt);
 	towire_s64(pptr, inflight->splice_amnt);
-	towire_u8(pptr, inflight->last_tx ? 1 : 0);
-	if(inflight->last_tx) {
-		towire_bitcoin_tx(pptr, inflight->last_tx);
-		towire_bitcoin_signature(pptr, &inflight->last_sig);
-	}
+	towire_bitcoin_tx(pptr, inflight->last_tx);
+	towire_bitcoin_signature(pptr, &inflight->last_sig);
 	towire_bool(pptr, inflight->i_am_initiator);
-	towire_bool(pptr, inflight->force_sign_first);
-	towire_u8(pptr, inflight->locked_scid ? 1 : 0);
-	if (inflight->locked_scid)
-		towire_short_channel_id(pptr, *inflight->locked_scid);
+}
+
+void copy_inflight(struct inflight *dest, struct inflight *src)
+{
+	dest->outpoint = src->outpoint;
+	dest->amnt = src->amnt;
+	dest->psbt = src->psbt ? clone_psbt(dest, src->psbt): NULL;
+	dest->splice_amnt = src->splice_amnt;
+	dest->last_tx = src->last_tx ? clone_bitcoin_tx(dest, src->last_tx) : NULL;
+	dest->last_sig = src->last_sig;
+	dest->i_am_initiator = src->i_am_initiator;
 }