Fix dockerfile build

.circleci/config.yml

@@ -0,0 +1,144 @@
+version: 2
+jobs:
+  # publish jobs require $DOCKERHUB_REPO, $DOCKERHUB_USER, $DOCKERHUB_PASS defined
+  amd64:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          command: |
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            #
+            sudo docker build --pull -t $DOCKERHUB_REPO:$LATEST_TAG-amd64 -f Dockerfile .
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-amd64
+  arm32v7:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          command: |
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            # Make sure the builder is copy the arm emulator
+            sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset
+            sed -i -e 's/#EnableQEMU //g' "contrib/linuxarm32v7.Dockerfile"
+            #
+            sudo docker build --pull -t $DOCKERHUB_REPO:$LATEST_TAG-arm32v7 -f contrib/linuxarm32v7.Dockerfile .
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-arm32v7
+  arm64v8:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          command: |
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            # Make sure the builder is copy the arm emulator
+            sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset
+            sed -i -e 's/#EnableQEMU //g' "contrib/linuxarm64v8.Dockerfile"
+            #
+            sudo docker build --pull -t $DOCKERHUB_REPO:$LATEST_TAG-arm64v8 -f contrib/linuxarm64v8.Dockerfile .
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-arm64v8
+  publish_docker_dev:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          command: |
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            #
+            sudo docker build --build-arg "DEVELOPER=1" --build-arg "TRACE_TOOLS=false" -t "$DOCKERHUB_REPO:$LATEST_TAG-dev" .
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-dev
+  publish_docker_bench:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          command: |
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            #
+            sudo docker build --build-arg "DEVELOPER=1" --build-arg "TRACE_TOOLS=true" -t "$DOCKERHUB_REPO:$LATEST_TAG-bench" .
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-bench
+  multiarch:
+    machine:
+      enabled: true
+      image: circleci/classic:201808-01
+    steps:
+      - run:
+          command: |
+            # Turn on Experimental features
+            sudo mkdir $HOME/.docker
+            sudo sh -c 'echo "{ \"experimental\": \"enabled\" }" >> $HOME/.docker/config.json'
+            #
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            #
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            sudo docker manifest create --amend $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-amd64 $DOCKERHUB_REPO:$LATEST_TAG-arm32v7 $DOCKERHUB_REPO:$LATEST_TAG-arm64v8
+            sudo docker manifest annotate $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-amd64 --os linux --arch amd64
+            sudo docker manifest annotate $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-arm32v7 --os linux --arch arm --variant v7
+            sudo docker manifest annotate $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-arm64v8 --os linux --arch arm64 --variant v8
+            sudo docker manifest push $DOCKERHUB_REPO:$LATEST_TAG -p
+
+workflows:
+  version: 2
+  publish:
+    jobs:
+      - amd64:
+          filters:
+            # ignore any commit on any branch by default
+            branches:
+              ignore: /.*/
+            # only act on version tags
+            tags:
+              only: /basedon-.+/
+      - arm32v7:
+          filters:
+            # ignore any commit on any branch by default
+            branches:
+              ignore: /.*/
+            # only act on version tags
+            tags:
+              only: /basedon-.+/
+      - arm64v8:
+          filters:
+            # ignore any commit on any branch by default
+            branches:
+              ignore: /.*/
+            # only act on version tags
+            tags:
+              only: /basedon-.+/
+      - publish_docker_dev:
+          filters:
+            # ignore any commit on any branch by default
+            branches:
+              ignore: /.*/
+            # only act on version tags
+            tags:
+              only: /basedon-.+/
+      - publish_docker_bench:
+          filters:
+            # ignore any commit on any branch by default
+            branches:
+              ignore: /.*/
+            # only act on version tags
+            tags:
+              only: /basedon-.+/
+      - multiarch:
+          requires:
+            - amd64
+            - arm32v7
+            - arm64v8
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /basedon-.+/
+

.dockerignore

@@ -1,3 +1,3 @@
 Dockerfile
-contrib/docker/Dockerfile.*
-target
+linuxarm32v7.Dockerfile
+.circleci/config.yml

.gitattributes

@@ -17,4 +17,4 @@ statements_gettextgen.po linguist-generated=true
 cln-grpc/proto/node.proto -text -diff linguist-generated=true
 cln-grpc/src/convert.rs -text -diff linguist-generated=true
 cln-rpc/src/model.rs -text -diff linguist-generated=true
-contrib/pyln-testing/pyln/testing/node_pb2.py -text -diff linguist-generated=true
+contrib/pyln-testing/pyln/testing/node_pb2.py -text -diff linguist-generated=true

.github/CODEOWNERS

@@ -13,7 +13,11 @@ contrib/msggen/         @cdecker
 contrib/pyln-client/    @cdecker
 contrib/pyln-testing/   @cdecker
 # Needed to ensure hsmd wire compatibility between releases
-hsmd/hsmd_wire.csv      @cdecker @ksedgwic @devrandom
+hsmd/hsmd_wire.csv      @cdecker
+
+wallet/invoices.*          @ZmnSCPxj
+plugins/multifundchannel.c @ZmnSCPxj
+doc/BACKUP.md              @ZmnSCPxj
 
 # See https://help.github.com/articles/about-codeowners/ for more
 # information

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,16 +0,0 @@
-> [!IMPORTANT]
->
-> 25.09 FREEZE July 28TH: Non-bugfix PRs not ready by this date will wait for 25.12.
->
-> RC1 is scheduled on _August 11th_
->
-> The final release is scheduled for September 1st.
-
-
-## Checklist
-Before submitting the PR, ensure the following tasks are completed. If an item is not applicable to your PR, please mark it as checked:
-
-- [ ] The changelog has been updated in the relevant commit(s) according to the [guidelines](https://docs.corelightning.org/docs/coding-style-guidelines#changelog-entries-in-commit-messages).
-- [ ] Tests have been added or modified to reflect the changes.
-- [ ] Documentation has been reviewed and updated as needed.
-- [ ] Related issues have been listed and linked, including any that this PR closes.

.github/scripts/build.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+set -e
+echo "Running in $(pwd)"
+export ARCH=${ARCH:-64}
+export BOLTDIR=bolts
+export CC=${COMPILER:-gcc}
+export COMPAT=${COMPAT:-1}
+export TEST_CHECK_DBSTMTS=${TEST_CHECK_DBSTMTS:-0}
+export DEVELOPER=${DEVELOPER:-1}
+export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0}
+export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
+export PYTEST_OPTS="--maxfail=5 --suppress-no-test-exit-code ${PYTEST_OPTS}"
+export PYTEST_PAR=${PYTEST_PAR:-10}
+export PYTEST_SENTRY_ALWAYS_REPORT=1
+export SLOW_MACHINE=1
+export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"}
+export TEST_DB_PROVIDER=${TEST_DB_PROVIDER:-"sqlite3"}
+export TEST_NETWORK=${NETWORK:-"regtest"}
+export TIMEOUT=900
+export VALGRIND=${VALGRIND:-0}
+export FUZZING=${FUZZING:-0}
+export LIGHTNINGD_POSTGRES_NO_VACUUM=1
+
+# Fail if any commands fail.
+set -e
+
+pip3 install --user poetry
+poetry config virtualenvs.create false --local
+poetry install
+
+git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
+git submodule update --init --recursive
+
+./configure CC="$CC"
+cat config.vars
+
+cat << EOF > pytest.ini
+[pytest]
+addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42
+markers =
+    slow_test: marks tests as slow (deselect with '-m "not slow_test"')
+EOF
+
+if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ]
+then
+    export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/
+    export MAKE_HOST="$TARGET_HOST"
+    export BUILD=x86_64-pc-linux-gnu
+    export AR="$TARGET_HOST"-ar
+    export AS="$TARGET_HOST"-as
+    export CC="$TARGET_HOST"-gcc
+    export CXX="$TARGET_HOST"-g++
+    export LD="$TARGET_HOST"-ld
+    export STRIP="$TARGET_HOST"-strip
+    export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static
+
+    wget -q https://zlib.net/zlib-1.2.12.tar.gz
+    tar xf zlib-1.2.12.tar.gz
+    cd zlib-1.2.12 || exit 1
+    ./configure --prefix="$QEMU_LD_PREFIX"
+    make
+    sudo make install
+    cd .. || exit 1
+    rm zlib-1.2.12.tar.gz && rm -rf zlib-1.2.12
+
+    wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip
+    unzip -q sqlite-src-3260000.zip
+    cd sqlite-src-3260000 || exit 1
+    automake --add-missing --force-missing --copy || true
+    ./configure --disable-tcl \
+     --enable-static \
+     --disable-readline \
+     --disable-threadsafe \
+     --disable-load-extension \
+     --host="$TARGET_HOST" \
+     --prefix="$QEMU_LD_PREFIX"
+    make
+    sudo make install
+    cd .. || exit 1
+    rm sqlite-src-3260000.zip
+    rm -rf sqlite-src-3260000
+
+    wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz
+    tar xf gmp-6.1.2.tar.xz
+    cd gmp-6.1.2 || exit 1
+    ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST"
+    make
+    sudo make install
+    cd ..
+    rm gmp-6.1.2.tar.xz
+    rm -rf gmp-6.1.2
+
+    ./configure CC="$TARGET_HOST-gcc" --enable-static
+
+    make -s -j32 CC="$TARGET_HOST-gcc"
+else
+    eatmydata make -j32
+    # shellcheck disable=SC2086
+    eatmydata $TEST_CMD
+fi

.github/scripts/install-bitcoind.sh

@@ -1,31 +0,0 @@
-#!/bin/sh
-
-set -e
-
-export BITCOIN_VERSION=27.1
-export ELEMENTS_VERSION=23.2.1
-
-DIRNAME="bitcoin-${BITCOIN_VERSION}"
-EDIRNAME="elements-${ELEMENTS_VERSION}"
-FILENAME="${DIRNAME}-x86_64-linux-gnu.tar.gz"
-EFILENAME="${EDIRNAME}-x86_64-linux-gnu.tar.gz"
-
-cd /tmp/
-
-# Since we inadvertently broke `elementsd` support in the past we only
-# want to download and enable the daemon that is actually going to be
-# used when running in CI. Otherwise we could end up accidentally
-# testing against `bitcoind` but still believe that we ran against
-# `elementsd`.
-if [ "$TEST_NETWORK" = "liquid-regtest" ]; then
-    wget "https://github.com/ElementsProject/elements/releases/download/elements-${ELEMENTS_VERSION}/${EFILENAME}"
-    tar -xf "${EFILENAME}"
-    sudo mv "${EDIRNAME}"/bin/* "/usr/local/bin"
-    rm -rf "${EFILENAME}" "${EDIRNAME}"
-else
-    wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/${FILENAME}"
-    tar -xf "${FILENAME}"
-    sudo mv "${DIRNAME}"/bin/* "/usr/local/bin"
-    rm -rf "${FILENAME}" "${DIRNAME}"
-fi
-

.github/scripts/setup.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 export DEBIAN_FRONTEND=noninteractive
+export BITCOIN_VERSION=0.20.1
+export ELEMENTS_VERSION=0.18.1.8
 export RUST_VERSION=stable
 
 sudo useradd -ms /bin/bash tester
@@ -20,18 +22,12 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      gcc-arm-none-eabi \
      gettext \
      git \
-     gnupg \
-     jq \
      libc6-dev-arm64-cross \
      libc6-dev-armhf-cross \
-     libev-dev \
-     libevent-dev \
-     libffi-dev \
-     libicu-dev \
+     libgmp-dev \
      libpq-dev \
      libprotobuf-c-dev \
      libsqlite3-dev \
-     libssl-dev \
      libtool \
      libxml2-utils \
      locales \
@@ -49,45 +45,31 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      software-properties-common \
      sudo \
      tcl \
-     tclsh \
      unzip \
      valgrind \
      wget \
      xsltproc \
-     systemtap-sdt-dev \
      zlib1g-dev
 
 echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
 sudo chmod 0440 /etc/sudoers.d/tester
 
-"$(dirname "$0")"/install-bitcoind.sh
+(
+    cd /tmp/ || exit 1
+    wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2
+    wget -q https://storage.googleapis.com/c-lightning-tests/elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2
+    tar -xjf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2
+    tar -xjf elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2
+    sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin
+    sudo mv elements-$ELEMENTS_VERSION/bin/* /usr/local/bin
+    rm -rf \
+       bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz \
+       bitcoin-$BITCOIN_VERSION \
+       elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 \
+       elements-$ELEMENTS_VERSION
+)
 
-curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
-     -y --default-toolchain ${RUST_VERSION}
-
-# We also need a relatively recent protobuf-compiler, at least 3.12.0,
-# in order to support the experimental `optional` flag.
-
-# BUT WAIT!  Gentoo wants this to match the version from the Python protobuf,
-# which comes from the same tree.  Makes sense!
-
-# And
-#   grpcio-tools-1.69.0` requires `protobuf = ">=5.26.1,<6.0dev"`
-
-# Now, protoc changed to date-based releases, BUT Python protobuf
-# didn't, so Python protobuf 4.21.12 (in Ubuntu 23.04) corresponds to
-# protoc 21.12 (which, FYI, is packaged in Ubuntu as version 3.21.12).
-
-# In general protobuf version x.y.z corresponds to protoc version y.z
-
-# Honorable mention go to Matt Whitlock for spelunking this horror with me!
-
-PROTOC_VERSION=29.4
-PB_REL="https://github.com/protocolbuffers/protobuf/releases"
-curl -LO $PB_REL/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip
-sudo unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local/
-sudo chmod a+x /usr/local/bin/protoc
-export PROTOC=/usr/local/bin/protoc
-export PATH=$PATH:/usr/local/bin
-env
-ls -lha /usr/local/bin
+if [ "$RUST" == "1" ]; then
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
+      -y --default-toolchain ${RUST_VERSION}
+fi

.github/scripts/sync-rpc-cmds.py

@@ -1,115 +0,0 @@
-import os
-from time import sleep
-import requests
-import re
-from enum import Enum
-
-# readme url
-URL = "https://dash.readme.com/api/v1"
-# category id for API reference
-CATEGORY_ID = "63e4e160c60b2e001dd1cc4e"
-CATEGORY_SLUG = "json-rpc-apis"
-
-
-class Action(Enum):
-    ADD = 'add'
-    UPDATE = 'update'
-    DELETE = 'delete'
-
-
-def getListOfRPCDocs(headers):
-    response = requests.get(f"{URL}/categories/{CATEGORY_SLUG}/docs", headers=headers)
-    if response.status_code == 200:
-        return response.json()
-    else:
-        return []
-
-
-def publishDoc(action, title, body, order, headers):
-    payload = {
-        "title": title,
-        "type": "basic",
-        "body": body,
-        "category": CATEGORY_ID,
-        "hidden": False,
-        "order": order,
-    }
-    # title == slug
-    if action == Action.ADD:
-        # create doc
-        response = requests.post(URL + "/docs", json=payload, headers=headers)
-        if response.status_code != 201:
-            print(response.text)
-        else:
-            print("Created ", title)
-    elif action == Action.UPDATE:
-        # update doc
-        response = requests.put(f"{URL}/docs/{title}", json=payload, headers=headers)
-        if response.status_code != 200:
-            print(response.text)
-        else:
-            print("Updated ", title)
-    elif action == Action.DELETE:
-        # delete doc
-        response = requests.delete(f"{URL}/docs/{title}", headers=headers)
-        if response.status_code != 204:
-            print(response.text)
-        else:
-            print("Deleted ", title)
-    else:
-        print("Invalid action")
-
-
-def extract_rpc_commands(rst_content):
-    manpages_block = re.search(
-        r"\.\. block_start manpages(.*?)" r"\.\. block_end manpages",
-        rst_content,
-        re.DOTALL,
-    )
-    if manpages_block:
-        commands = re.findall(
-            r"\b([a-zA-Z0-9_-]+)" r"\s+<([^>]+)>\n", manpages_block.group(1)
-        )
-        return commands
-    return []
-
-
-def main():
-    # define headers for requests
-    headers = {
-        "accept": "application/json",
-        "content-type": "application/json",
-        "authorization": "Basic " + os.environ.get("README_API_KEY"),
-    }
-
-    # path to the rst file from where we fetch all the RPC commands
-    path_to_rst = "doc/index.rst"
-    with open(path_to_rst, "r") as file:
-        rst_content = file.read()
-
-    commands_from_local = extract_rpc_commands(rst_content)
-    commands_from_readme = getListOfRPCDocs(headers)
-
-    # Compare local and server commands list to get the list of command to add or delete
-    commands_local_title = set(command[0] for command in commands_from_local)
-    commands_readme_title = set(command['title'] for command in commands_from_readme)
-    commands_to_delete = commands_readme_title - commands_local_title
-    commands_to_add = commands_local_title - commands_readme_title
-    for name in commands_to_delete:
-        publishDoc(Action.DELETE, name, "", 0, headers)
-        sleep(3)
-
-    if commands_from_local:
-        order = 0
-        for name, file in commands_from_local:
-            with open("doc/" + file) as f:
-                body = f.read()
-                publishDoc(Action.ADD if name in commands_to_add else Action.UPDATE, name, body, order, headers)
-            order = order + 1
-            sleep(3)
-    else:
-        print("No commands found in the Manpages block.")
-
-
-if __name__ == "__main__":
-    main()

.github/workflows/bsd.yml

@@ -5,66 +5,76 @@ on:
     branches:
       - "master"
   pull_request:
-  workflow_dispatch:
 
 jobs:
   testfreebsd:
-    runs-on: ubuntu-22.04
+    runs-on: macos-10.15
     name: Build and test on FreeBSD
-    timeout-minutes: 120
-    strategy:
-      fail-fast: true
-      matrix:
-        bitcoind-version: ["27.1"]
+    env:
+      DEVELOPER: 1
+      VALGRIND: 0
+      EXPERIMENTAL_FEATURES: 0
+      COMPAT: 1
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v2
     - name: Test in FreeBSD
       id: test
-      uses: vmactions/freebsd-vm@v1
+      uses: vmactions/freebsd-vm@v0.1.5
       with:
+        envs: 'DEVELOPER VALGRIND EXPERIMENTAL_FEATURES COMPAT'
         usesh: true
-        sync: rsync
-        copyback: false
         prepare: |
           pkg install -y \
-            bash \
             wget \
-            python310 \
+            py38-pip \
+            py38-sqlite3 \
             gmake \
             git \
             python \
-            postgresql16-server \
+            postgresql12-server \
             autoconf \
             automake \
             libtool \
+            gmp \
             bash \
             gettext \
             sqlite3 \
             lowdown \
-            pkgconf \
-            jq \
-            protobuf \
             curl
 
-            python3.10 -m ensurepip
-            python3.10 -m pip install --upgrade pip
-            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
-
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2021-08-3z1
 
+          cd /tmp/ || exit 1
+          wget https://storage.googleapis.com/c-lightning-tests/bitcoin-0.20.1-x86_64-linux-gnu.tar.bz2
+          tar -xjf bitcoin-0.20.1-x86_64-linux-gnu.tar.bz2
+          sudo mv bitcoin-0.20.1/bin/* /usr/local/bin
+          rm -rf \
+            bitcoin-0.20.1-x86_64-linux-gnu.tar.gz \
+            bitcoin-0.20.1
+          
         run: |
-          PATH=/root/.local/bin:$PATH:/root/.cargo/bin; export PATH
+          PATH=/root/.local/bin:$PATH
           pip install --user -U wheel pip
-          pip3 install --user poetry
-          poetry install
+          pip install --user -U -r requirements.txt
+
+          # Install utilities that aren't dependencies, but make
+          # running tests easier/feasible on CI (and pytest which
+          # keeps breaking the rerunfailures plugin).
+          pip install --user -U \
+            blinker \
+            flake8 \
+            mako \
+            pytest-sentry \
+            pytest-test-groups==1.0.3 \
+            pytest-custom-exit-code==0.3.0 \
+            pytest-json-report
 
           git clone https://github.com/lightning/bolts.git ../bolts
           # fatal: unsafe repository ('/Users/runner/work/lightning/lightning' is owned by someone else)
           git config --global --add safe.directory `pwd`
-          for d in libsodium libwally-core gheap jsmn libbacktrace lowdown; do git config --global --add safe.directory `pwd`/external/$d; done
           git submodule update --init --recursive
 
-          ./configure CC="$CC" --disable-valgrind
-
+          ./configure CC="$CC"
           cat config.vars
 
           cat << EOF > pytest.ini
@@ -75,5 +85,8 @@ jobs:
           EOF
 
           # Just run a "quick" test without memory checking
-          poetry run gmake
+          gmake
+
+          # Clean up to maximize rsync's chances of succeeding
+          gmake clean
 

.github/workflows/ci.yaml

@@ -5,649 +5,363 @@ on:
     branches:
       - "master"
   pull_request:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-env:
-  # Makes the upload-artifact work more reliably at the cost
-  # of a bit of compile time.
-  RUST_PROFILE: release
-  SLOW_MACHINE: 1
-  CI_SERVER_URL: "http://35.239.136.52:3170"
-
 jobs:
-  prebuild:
-    name: Pre-build checks
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    env:
-      BOLTDIR: bolts
-    strategy:
-      fail-fast: true
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
-
-      - name: Rebase
-        # We can't rebase if we're on master already.
-        if: github.ref != 'refs/heads/master'
-        run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-          git fetch origin ${{ github.base_ref }}
-          git rebase origin/${{ github.base_ref }}
-
-      - name: Check changelog
-        env:
-          PR_DESCRIPTION: "${{ github.event.pull_request.body || '' }}"
-          EVENT_NAME: "${{ github.event_name }}"
-          BASE_REF: "${{ github.base_ref || 'master' }}"
-        run: |
-          echo "Event Name: $EVENT_NAME"
-          echo "Base Ref: $BASE_REF"
-          echo "PR DESCRIPTION: $PR_DESCRIPTION"
-          if [ "$EVENT_NAME" = "pull_request" ]; then
-            if [[ "$PR_DESCRIPTION" != *"Changelog-None"* && \
-                  -z "$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')" && \
-                  "$(git rev-parse --abbrev-ref HEAD)" != "$BASE_REF" ]]; then
-              echo "::error::'Changelog' entry is missing in all commits, and 'Changelog-None' not specified in the PR description"
-              exit 1
-            else
-              if [[ "$PR_DESCRIPTION" == *"Changelog-None"* ]]; then
-                echo "Changelog found in PR description"
-              else
-                echo "Changelog found in Commit \"$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')\""
-              fi
-            fi
-          else
-            echo "Not a PR event."
-          fi
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          # Export and then use pip to install into the current env
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-          # We're going to check BOLT quotes, so get the latest version
-          git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
-      - name: Configure
-        run: ./configure --enable-debugbuild --enable-rust
-      - name: Check source
-        env:
-          VALGRIND: 0
-          PYTEST_OPTS: --timeout=1200
-        run: make check-source BASE_REF="origin/${{ github.base_ref }}"
-      - name: Check Generated Files have been updated
-        run: make check-gen-updated
-      - name: Check docs
-        run: make check-doc
-
-  compile:
-    name: Compile CLN ${{ matrix.cfg }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    needs:
-      - prebuild
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - CFG: compile-gcc
-            VALGRIND: 1
-            COMPILER: gcc
-          - CFG: compile-gcc-O3
-            VALGRIND: 1
-            COMPILER: gcc
-            COPTFLAGS_VAR: COPTFLAGS="-O3 -Werror"
-          # While we're at it let's try to compile with clang
-          - CFG: compile-clang
-            VALGRIND: 1
-            COMPILER: clang
-          - CFG: compile-clang-sanitizers
-            COMPILER: clang
-            ASAN: 1
-            UBSAN: 1
-            VALGRIND: 0
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-
-      - name: Build
-        env:
-          COMPILER: ${{ matrix.COMPILER }}
-          ASAN: ${{ matrix.ASAN }}
-          UBSAN: ${{ matrix.UBSAN }}
-          VALGRIND: ${{ matrix.VALGRIND }}
-          COMPAT: 1
-          CFG: ${{ matrix.CFG }}
-        run: |
-          set -e
-          pip3 install --user pip wheel poetry
-          poetry self add poetry-plugin-export
-          poetry export -o requirements.txt --with dev --without-hashes
-          python3 -m pip install -r requirements.txt
-          ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }}
-
-          make -j $(nproc) testpack.tar.bz2
-
-          # Rename now so we don't clash
-          mv testpack.tar.bz2 cln-${CFG}.tar.bz2
-      - name: Check rust packages
-        run: cargo test --all
-      - uses: actions/upload-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-          path: cln-${{ matrix.CFG }}.tar.bz2
-
-  check-units:
-    # The unit test checks are not in the critical path (not dependent
-    # on the integration tests), so run them with `valgrind`
-    name: Run unit tests
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    env:
-      BOLTDIR: bolts
-    needs:
-      - compile
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - CFG: compile-gcc
-            VALGRIND: 1
-          - CFG: compile-clang-sanitizers
-            VALGRIND: 0
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          sudo apt-get update -qq
-          sudo apt-get install -y -qq lowdown
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          # Export and then use pip to install into the current env
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-          # We're going to check BOLT quotes, so get the latest version
-          git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-
-      - name: Check
-        run: |
-          tar -xaf cln-${{ matrix.CFG }}.tar.bz2
-          eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }}
-
-  check-fuzz:
-    name: Run fuzz regression tests
-    runs-on: ubuntu-22.04
-    needs:
-      - prebuild
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          # Export and then use pip to install into the current env
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-
-      - name: Build
-        run: |
-          ./configure --enable-debugbuild --enable-fuzzing --enable-address-sanitizer --enable-ub-sanitizer --disable-valgrind CC=clang
-          make -j $(nproc) check-fuzz
-
-  integration:
-    name: Test CLN ${{ matrix.name }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    env:
-      RUST_PROFILE: release  # Has to match the one in the compile step
-      PYTEST_OPTS: --timeout=1200
-    needs:
-      - compile
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - NAME: gcc
-            CFG: compile-gcc
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: gcc
-            TEST_NETWORK: regtest
-          # While we're at it let's try to compile with clang
-          - NAME: clang
-            CFG: compile-clang
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: clang
-            TEST_NETWORK: regtest
-          # And of course we want to test postgres too
-          - NAME: postgres
-            CFG: compile-gcc
-            COMPILER: gcc
-            TEST_DB_PROVIDER: postgres
-            TEST_NETWORK: regtest
-          # And don't forget about elements (like cdecker did when
-          # reworking the CI...)
-          - NAME: liquid
-            CFG: compile-gcc
-            COMPILER: gcc
-            TEST_NETWORK: liquid-regtest
-            TEST_DB_PROVIDER: sqlite3
-          # And dual funding!
-          - NAME: dual-fund
-            CFG: compile-gcc
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: gcc
-            TEST_NETWORK: regtest
-            EXPERIMENTAL_DUAL_FUND: 1
-          # And splicing!
-          - NAME: splicing
-            CFG: compile-gcc
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: gcc
-            TEST_NETWORK: regtest
-            EXPERIMENTAL_SPLICING: 1
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          pip3 install --user pip wheel poetry
-          poetry install
-
-      - name: Install bitcoind
-        env:
-          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
-        run: .github/scripts/install-bitcoind.sh
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-
-      - name: Unpack pre-built CLN
-        env:
-          CFG: ${{ matrix.CFG }}
-        run: |
-          tar -xaf cln-${CFG}.tar.bz2
-
-      - name: Switch network
-        if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }}
-        run: |
-          # Loading the network from config.vars rather than the envvar is a terrible idea...
-          sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars
-          cat config.vars
-
-      - name: Test
-        env:
-          COMPILER: ${{ matrix.COMPILER }}
-          EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }}
-          EXPERIMENTAL_SPLICING: ${{ matrix.EXPERIMENTAL_SPLICING }}
-          COMPAT: 1
-          CFG: ${{ matrix.CFG }}
-          SLOW_MACHINE: 1
-          PYTEST_PAR: 10
-          TEST_DEBUG: 1
-          TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
-          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
-          LIGHTNINGD_POSTGRES_NO_VACUUM: 1
-        run: |
-          env
-          cat config.vars
-          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
-
-  integration-valgrind:
-    name: Valgrind Test CLN ${{ matrix.name }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    env:
-      RUST_PROFILE: release  # Has to match the one in the compile step
-      CFG: compile-gcc
-      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800
-    needs:
-      - compile
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - NAME: Valgrind (01/10)
-            PYTEST_OPTS: --test-group=1  --test-group-count=10
-          - NAME: Valgrind (02/10)
-            PYTEST_OPTS: --test-group=2  --test-group-count=10
-          - NAME: Valgrind (03/10)
-            PYTEST_OPTS: --test-group=3  --test-group-count=10
-          - NAME: Valgrind (04/10)
-            PYTEST_OPTS: --test-group=4  --test-group-count=10
-          - NAME: Valgrind (05/10)
-            PYTEST_OPTS: --test-group=5  --test-group-count=10
-          - NAME: Valgrind (06/10)
-            PYTEST_OPTS: --test-group=6  --test-group-count=10
-          - NAME: Valgrind (07/10)
-            PYTEST_OPTS: --test-group=7  --test-group-count=10
-          - NAME: Valgrind (08/10)
-            PYTEST_OPTS: --test-group=8  --test-group-count=10
-          - NAME: Valgrind (09/10)
-            PYTEST_OPTS: --test-group=9  --test-group-count=10
-          - NAME: Valgrind (10/10)
-            PYTEST_OPTS: --test-group=10 --test-group-count=10
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          sudo apt-get update -qq
-          sudo apt-get install -yyq valgrind
-          pip3 install --user pip wheel poetry
-          poetry install
-
-      - name: Install bitcoind
-        run: .github/scripts/install-bitcoind.sh
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-compile-gcc.tar.bz2
-
-      - name: Unpack build
-        run: tar -xvjf cln-compile-gcc.tar.bz2
-
-      - name: Test
-        env:
-          SLOW_MACHINE: 1
-          TEST_DEBUG: 1
-        run: |
-          VALGRIND=1 poetry run eatmydata pytest tests/ -vvv -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
-
-  integration-sanitizers:
-    name: Sanitizers Test CLN
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    env:
-      RUST_PROFILE: release
-      SLOW_MACHINE: 1
-      TEST_DEBUG: 1
-      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800
-    needs:
-      - compile
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - NAME: ASan/UBSan (01/10)
-            PYTEST_OPTS: --test-group=1  --test-group-count=10
-          - NAME: ASan/UBSan (02/10)
-            PYTEST_OPTS: --test-group=2  --test-group-count=10 -n 1
-          - NAME: ASan/UBSan (03/10)
-            PYTEST_OPTS: --test-group=3  --test-group-count=10
-          - NAME: ASan/UBSan (04/10)
-            PYTEST_OPTS: --test-group=4  --test-group-count=10
-          - NAME: ASan/UBSan (05/10)
-            PYTEST_OPTS: --test-group=5  --test-group-count=10
-          - NAME: ASan/UBSan (06/10)
-            PYTEST_OPTS: --test-group=6  --test-group-count=10
-          - NAME: ASan/UBSan (07/10)
-            PYTEST_OPTS: --test-group=7  --test-group-count=10
-          - NAME: ASan/UBSan (08/10)
-            PYTEST_OPTS: --test-group=8  --test-group-count=10
-          - NAME: ASan/UBSan (09/10)
-            PYTEST_OPTS: --test-group=9  --test-group-count=10
-          - NAME: ASan/UBSan (10/10)
-            PYTEST_OPTS: --test-group=10 --test-group-count=10
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          set -e
-          pip3 install --user wheel poetry
-          poetry install --with dev --no-root
-
-      - name: Install bitcoind
-        run: .github/scripts/install-bitcoind.sh
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-compile-clang-sanitizers.tar.bz2
-
-      - name: Unpack build
-        run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2
-
-      - name: Test
-        run: |
-          poetry run eatmydata pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
-
-  update-docs-examples:
-    name: Update examples in doc schemas (disabled temporarily!)
-    if: false
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
+  smoke-test:
+    name: Smoke Test ${{ matrix.cfg }}
+    runs-on: ubuntu-20.04
+    timeout-minutes: 300
     env:
+      DEVELOPER: 1
       VALGRIND: 0
-      GENERATE_EXAMPLES: 1
-      PYTEST_OPTS: --timeout=1200
-      TEST_NETWORK: regtest
-    needs:
-      - compile
+      EXPERIMENTAL_FEATURES: 0
+      COMPAT: 1
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - CFG: "make and unit test w/ VALGRIND"
+            TEST_CMD: "make default check-source"
+            VALGRIND: 1
+          - CFG: "make-O3-check"
+            TEST_CMD: "make check-source check-units installcheck check-gen-updated"
+            COPTFLAGS: "-O3"
+          - CFG: "make-32-bit-nodev-check"
+            ARCH: 32
+            TEST_CMD: "make check-source check-units installcheck"
+            DEVELOPER: 0
+          - CFG: "make-EXPERIMENTAL-check"
+            TEST_CMD: "make check-source check-units installcheck check-gen-updated"
+            EXPERIMENTAL_FEATURES: 1
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+        uses: actions/checkout@v2.0.0
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
         with:
-          python-version: '3.10'
+          python-version: 3.7
+
       - name: Install dependencies
         run: |
           bash -x .github/scripts/setup.sh
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-      - name: Install bitcoind
-        env:
-          TEST_NETWORK: regtest
-        run: .github/scripts/install-bitcoind.sh
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-compile-gcc.tar.bz2
-      - name: Unpack pre-built CLN
-        run: |
-          tar -xaf cln-compile-gcc.tar.bz2
-      - name: Test
-        run: |
-          eatmydata make -j $(nproc) check-doc-examples
 
-  min-btc-support:
-    name: Test minimum supported BTC v${{ matrix.MIN_BTC_VERSION }} with ${{ matrix.NAME }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
+      - name: Build
+        env:
+          VALGRIND: ${{ matrix.VALGRIND }}
+          DEVELOPER: ${{ matrix.DEVELOPER }}
+          EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }}
+          COMPILER: ${{ matrix.COMPILER }}
+          ARCH: ${{ matrix.ARCH }}
+          COMPAT: ${{ matrix.COMPAT }}
+          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
+          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
+          COPTFLAGS: ${{ matrix.COPTFLAGS }}
+          NETWORK: ${{ matrix.NETWORK }}
+          TEST_CMD: ${{ matrix.TEST_CMD }}
+          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
+          TEST_GROUP: ${{ matrix.TEST_GROUP }}
+        run: |
+          bash -x .github/scripts/build.sh
+
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Junit Report ${{ github.run_number }}.${{ matrix.cfg }}
+          path: report.*
+          if-no-files-found: ignore
+
+  check-dock:
+    name: Check core-lightning doc
+    runs-on: ubuntu-20.04
     env:
-      RUST_PROFILE: release  # Has to match the one in the compile step
-      PYTEST_OPTS: --timeout=1200
-    needs:
-      - compile
+      DEVELOPER: 1
+      VALGRIND: 0
+      EXPERIMENTAL_FEATURES: 0
+      COMPAT: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2.0.0
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Install dependencies
+        run: bash -x .github/scripts/setup.sh
+
+      - name: Check Doc
+        run: |
+          pip install mako
+          ./configure
+          make check-doc
+
+  proto-test:
+    name: Protocol Test Config
+    runs-on: ubuntu-22.04
+    timeout-minutes: 300
+    needs: [smoke-test]
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - {compiler: clang, db: sqlite3}
+          - {compiler: gcc, db: postgres}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2.0.0
+      - name: Build and run
+        run: |
+          docker build -f contrib/docker/Dockerfile.ubuntu -t cln-ci-ubuntu .
+          docker run -e ARCH=${{ matrix.arch }} \
+          -e COMPILER=${{ matrix.compiler }} \
+          -e DB=${{ matrix.db }} \
+          -e NETWORK=${{ matrix.network }} \
+          -e TARGET_HOST=${{ matrix.TARGET_HOST }} \
+          -e VALGRIND=${{ matrix.valgrind }} \
+          -e DEVELOPER=1 \
+          -e EXPERIMENTAL_FEATURES=1 \
+          -e COMPAT=0 \
+          -e PYTEST_PAR=2 \
+          -e PYTEST_OPTS="--timeout=300" \
+          -e TEST_CMD="make check-protos" \
+          -e TEST_GROUP=1 \
+          -e TEST_GROUP_COUNT=1 \
+          cln-ci-ubuntu
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Junit Report ${{ github.run_number }}.{{ matrix.cfg }}
+          path: report.*
+
+  normal-test:
+    name: Normal Test Config ${{ matrix.cfg }}
+    runs-on: ubuntu-20.04
+    needs: [smoke-test]
+    env:
+      DEVELOPER: 1
+      VALGRIND: 0
+      EXPERIMENTAL_FEATURES: 0
+      COMPAT: 1
     strategy:
       fail-fast: false
       matrix:
         include:
-          - NAME: clang
-            CFG: compile-clang
-            TEST_DB_PROVIDER: sqlite3
+          # All of the following will just run `make pytest`
+          - CFG: "clang-fuzzing"
             COMPILER: clang
-            TEST_NETWORK: regtest
-            MIN_BTC_VERSION: '25.0'
+            FUZZING: 1
+          - CFG: "check-dbstmts"
+            COMPILER: gcc
+            TEST_CHECK_DBSTMTS: 1
+          - CFG: "non-DEVELOPER-non-COMPAT-1"
+            DEVELOPER: 0
+            COMPAT: 0
+            TEST_GROUP: 1
+            TEST_GROUP_COUNT: 2
+          - CFG: "non-DEVELOPER-non-COMPAT-2"
+            DEVELOPER: 0
+            COMPAT: 0
+            TEST_GROUP: 2
+            TEST_GROUP_COUNT: 2
+          - CFG: "DUAL_FUND"
+            EXPERIMENTAL_DUAL_FUND: 1
+            DEVELOPER: 1
+            COMPAT: 0
+          # Various other configurations
+          - CFG: "Elements"
+            NETWORK: liquid-regtest
+          - CFG: "PostgreSQL"
+            DB: postgres
+            PYTEST_PAR: 2
+
+          # The cross-compiled versions
+          - CFG: "cross-arm32"
+            ARCH: arm32v7
+            TARGET_HOST: arm-linux-gnueabihf
+          - CFG: "cross-arm64"
+            ARCH: arm64v8
+            TARGET_HOST: aarch64-linux-gnu
+
+          # The experimental feature test
+          - CFG: "EXPERIMENTAL"
+            EXPERIMENTAL_FEATURES: 1
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2.0.0
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
         with:
-          python-version: '3.10'
+          python-version: 3.7
 
       - name: Install dependencies
         run: |
-          pip3 install --user pip wheel poetry
-          poetry install
+          bash -x .github/scripts/setup.sh
 
-      - name: Download Bitcoin Core
-        run: wget "https://bitcoincore.org/bin/bitcoin-core-${{ matrix.MIN_BTC_VERSION }}/bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz"
-
-      - name: Extract Bitcoin Core
-        run: tar -xf "bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz"
-
-      - name: Move Bitcoin Core Binaries
-        run: sudo mv bitcoin-${{ matrix.MIN_BTC_VERSION }}/bin/* /usr/local/bin/
-
-      - name: Clean Up Downloaded Bitcoin
-        run: rm -rf "bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz" "bitcoin-${{ matrix.MIN_BTC_VERSION }}"
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-
-      - name: Unpack pre-built CLN
-        env:
-          CFG: ${{ matrix.CFG }}
-        run: |
-          tar -xaf cln-${CFG}.tar.bz2
-
-      - name: Test
+      - name: Build
         env:
+          VALGRIND: ${{ matrix.VALGRIND }}
+          DEVELOPER: ${{ matrix.DEVELOPER }}
+          EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }}
+          EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }}
           COMPILER: ${{ matrix.COMPILER }}
-          COMPAT: 1
-          CFG: ${{ matrix.CFG }}
-          SLOW_MACHINE: 1
-          PYTEST_PAR: 10
-          TEST_DEBUG: 1
-          TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
-          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
-          LIGHTNINGD_POSTGRES_NO_VACUUM: 1
+          ARCH: ${{ matrix.ARCH }}
+          COMPAT: ${{ matrix.COMPAT }}
+          FUZZING: ${{ matrix.FUZZING }}
+          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
+          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
+          NETWORK: ${{ matrix.NETWORK }}
+          TEST_CHECK_DBSTMTS: ${{ matrix.TEST_CHECK_DBSTMTS }}
+          TEST_CMD: ${{ matrix.TEST_CMD }}
+          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
+          TEST_GROUP: ${{ matrix.TEST_GROUP }}
+          TEST_DB_PROVIDER: ${{ matrix.DB }}
         run: |
-          env
-          cat config.vars
-          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
+          bash -x .github/scripts/build.sh
 
-  check-flake:
-    name: Check Nix Flake
-    runs-on: ubuntu-22.04
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Junit Report ${{ github.run_number }}.${{ matrix.cfg }}
+          path: report.*
+
+  valgrind-test:
+    name: Valgrind Test Config ${{ matrix.cfg }}
+    runs-on: ubuntu-20.04
+    needs: [smoke-test]
+    env:
+      DEVELOPER: 1
+      EXPERIMENTAL_FEATURES: 0
+      COMPAT: 1
+      TEST_GROUP_COUNT: 10
+      PYTEST_PAR: 3
+      LABEL: "Valgrind-test"
     strategy:
       fail-fast: true
+      matrix:
+        include:
+          - CFG: "valgrind-1"
+            VALGRIND: 1
+            TEST_GROUP: 1
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-2"
+            VALGRIND: 1
+            TEST_GROUP: 2
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-3"
+            VALGRIND: 1
+            TEST_GROUP: 3
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-4"
+            VALGRIND: 1
+            TEST_GROUP: 4
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-5"
+            VALGRIND: 1
+            TEST_GROUP: 5
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-6"
+            VALGRIND: 1
+            TEST_GROUP: 6
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-7"
+            VALGRIND: 1
+            TEST_GROUP: 7
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-8"
+            VALGRIND: 1
+            TEST_GROUP: 8
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-9"
+            VALGRIND: 1
+            TEST_GROUP: 9
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-10"
+            VALGRIND: 1
+            TEST_GROUP: 10
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: 'recursive'
-      - name: Check Nix flake inputs
-        uses: DeterminateSystems/flake-checker-action@v8
-      - name: Install Nix
-        uses: cachix/install-nix-action@V27
-        with:
-          nix_path: nixpkgs=channel:nixos-24.05
-      - name: Check flake
-        run: nix flake check .?submodules=1#
+        uses: actions/checkout@v2.0.0
 
-  gather:
-    # A dummy task that depends on the full matrix of tests, and
-    # signals successful completion. Used for the PR status to pass
-    # before merging.  Needs to run even if they failed!
-    name: CI completion
-    runs-on: ubuntu-22.04
-    needs:
-      - integration
-      - check-units
-      - integration-valgrind
-      - integration-sanitizers
-      - min-btc-support
-      - check-flake
-    if: ${{ always() }}
-    steps:
-      - name: Complete
-        env:
-          JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC FLAKE"
-          INTEGRATION: ${{ needs.integration.result }}
-          CHECK_UNITS: ${{ needs['check-units'].result }}
-          VALGRIND: ${{ needs['integration-valgrind'].result }}
-          SANITIZERS: ${{ needs['integration-sanitizers'].result }}
-          DOCS: ${{ needs['update-docs-examples'].result }}
-          BTC: ${{ needs['min-btc-support'].result }}
-          FLAKE: ${{ needs['check-flake'].result }}
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Install dependencies
         run: |
-          failed=""
-          for name in $JOB_NAMES; do
-            result="${!name}"
-            echo "$name: $result"
-            if [[ "$result" != "success" ]]; then
-              failed="yes"
-            fi
-          done
-          if [[ "$failed" == "yes" ]]; then
-            echo "One or more required jobs failed"
-            exit 1
-          fi
+          bash -x .github/scripts/setup.sh
+
+      - name: Build
+        env:
+          VALGRIND: ${{ matrix.VALGRIND }}
+          DEVELOPER: ${{ matrix.DEVELOPER }}
+          EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }}
+          EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }}
+          COMPILER: ${{ matrix.COMPILER }}
+          ARCH: ${{ matrix.ARCH }}
+          COMPAT: ${{ matrix.COMPAT }}
+          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
+          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
+          NETWORK: ${{ matrix.NETWORK }}
+          TEST_CMD: ${{ matrix.TEST_CMD }}
+          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
+          TEST_GROUP: ${{ matrix.TEST_GROUP }}
+        run: |
+          bash -x .github/scripts/build.sh
+
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Junit Report ${{ github.run_number }}.${{ matrix.cfg }}
+          path: report.*
+
+  rust-test:
+    name: Rust Test Config
+    runs-on: ubuntu-20.04
+    needs: [smoke-test]
+    env:
+      DEVELOPER: 1
+      RUST: 1
+      VALGRIND: 0
+      # Run only the rust tests, others are not impacted.
+      TEST_CMD: "make -j 8 && pytest -vvv tests/test_cln_rs.py"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2.0.0
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Install dependencies
+        run: |
+          bash -x .github/scripts/setup.sh
+
+      - name: Build
+        run: |
+          bash -x .github/scripts/build.sh
+
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Junit Report ${{ github.run_number }}.${{ matrix.cfg }}
+          path: report.*

.github/workflows/ci_build.yml

@@ -0,0 +1,17 @@
+name: CI Compilation testing
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { OS: alpine }
+    steps:
+      - uses: actions/checkout@v2
+      - name: Integration testing
+        run: |
+          docker build -f contrib/docker/Dockerfile.${{matrix.OS}} -t clightning-${{matrix.OS}} .

.github/workflows/crate-io.yml

@@ -1,30 +0,0 @@
----
-name: "Release Rust 🦀"
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - "master"
-
-jobs:
-  release_rust:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@stable
-      - uses: arduino/setup-protoc@v3
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './cln-rpc'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './plugins'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './cln-grpc'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true

.github/workflows/docker-release.yml

@@ -1,118 +0,0 @@
-name: Build and push multi-platform docker images
-
-on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      version:
-        description: 'Release version'
-        required: true
-
-      repository-name:
-        description: 'Docker repository name'
-        default: 'elementsproject'
-        required: false
-
-      platforms-to-build:
-        description: 'List of platforms to build'
-        default: 'linux/amd64,linux/arm64,linux/arm/v7'
-        required: false
-  
-      push-latest:
-        description: 'Push the latest tag also?'
-        default: 'false'
-        required: false
-
-jobs:
-  build:
-    runs-on: ubuntu-22.04
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-      with:
-        ref: ${{ github.ref }}  # Ensures the branch triggering the workflow is checked out
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-
-    - name: Setup Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
-    - name: Log in to Docker Hub
-      uses: docker/login-action@v3
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
-    - name: Set up values
-      id: set-values
-      run: |
-        if [[ "${{ github.event.inputs.version }}" != "" ]]; then
-          echo "Input version provided"
-          VERSION=${{ github.event.inputs.version }}
-        elif [[ ${{ github.ref_type }} == "tag" ]]; then
-          echo "This is a tag event"
-          VERSION=${{ github.ref_name }}
-        else
-          echo "No version provided and no tag found."
-          exit 1
-        fi
-        echo "VERSION=$VERSION" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.repository-name }}" != "" ]]; then
-          REPONAME=${{ github.event.inputs.repository-name }}
-        else
-          REPONAME="elementsproject"
-        fi
-        echo "REPONAME=$REPONAME" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.platforms-to-build }}" != "" ]]; then
-          PLATFORMS=${{ github.event.inputs.platforms-to-build }}
-        else
-          PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
-        fi
-        echo "PLATFORMS=$PLATFORMS" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.push-latest }}" == "true" ]] || 
-          ([[ "${{ github.ref_type }}" == "tag" ]] && [[ ! "$VERSION" =~ rc ]]); then
-          echo "Latest true"
-          PUSHLATEST="true"
-        else
-          echo "Latest false"
-          PUSHLATEST="false"
-        fi
-        echo "PUSHLATEST=$PUSHLATEST" >> $GITHUB_ENV
-
-        TAGS="$REPONAME/lightningd:$VERSION"
-        if [[ "$PUSHLATEST" == "true" ]]; then
-          TAGS="$TAGS,$REPONAME/lightningd:latest"
-        fi
-        echo "TAGS=$TAGS" >> $GITHUB_ENV
-
-    - name: Print GitHub Ref Values
-      run: |
-        echo "GITHUB REF TYPE: ${{ github.ref_type }}"
-        echo "GITHUB REF NAME: ${{ github.ref_name }}"
-        echo "EVENT INPUT VERSION: ${{ github.event.inputs.version }}"
-        echo "EVENT INPUT REPO: ${{ github.event.inputs.repository-name }}"
-        echo "EVENT INPUT PLATFORMS: ${{ github.event.inputs.platforms-to-build }}"
-        echo "EVENT INPUT PUSH LATEST: ${{ github.event.inputs.push-latest }}"
-        echo "ENV VERSION: ${{ env.VERSION }}"
-        echo "ENV REPO NAME: ${{ env.REPONAME }}"
-        echo "ENV PLATFORMS: ${{ env.PLATFORMS }}"
-        echo "ENV PUSH LATEST: ${{ env.PUSHLATEST }}"
-        echo "ENV TAGS: ${{ env.TAGS }}"
-
-    - name: Build and push Docker image
-      uses: docker/build-push-action@v5
-      with:
-        context: .
-        file: ./Dockerfile
-        push: true
-        platforms: ${{ env.PLATFORMS }}
-        tags: ${{ env.TAGS }}

.github/workflows/docker.yml

@@ -1,34 +0,0 @@
-name: Docker packaging
-on:
-  push:
-    tags:
-      - 'basedon-*'
-
-jobs:
-  main:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Environment variables
-        run: env
-      -
-        name: Create images
-        env:
-          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
-          DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }}
-          DOCKERHUB_REPO: ${{ vars.DOCKERHUB_REPO }}
-        shell: bash
-        run: |
-          LATEST_TAG=${GITHUB_REF#refs/tags/}
-          LATEST_TAG=${LATEST_TAG:8} #trim "basedon-" from tag
-          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USER" --password-stdin
-          docker buildx create --use
-          DOCKER_BUILDX_OPTS="--platform linux/amd64,linux/arm64,linux/arm/v7 --push"
-          docker buildx build $DOCKER_BUILDX_OPTS -t $DOCKERHUB_REPO:$LATEST_TAG .

.github/workflows/macos.yaml

@@ -5,58 +5,64 @@ on:
 jobs:
   smoke-test:
     name: Smoke Test macOS
-    runs-on: macos-14
-    timeout-minutes: 120
+    runs-on: macos-latest
+    env:
+      DEVELOPER: 1
+      VALGRIND: 0
+      EXPERIMENTAL_FEATURES: 1
+      COMPAT: 0
     strategy:
       fail-fast: true
-      matrix:
-        bitcoind-version: ["27.1"]
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Download Bitcoin ${{ matrix.bitcoind-version }} & install binaries
-        run: |
-          export BITCOIND_VERSION=${{ matrix.bitcoind-version }}
-          export TARGET_ARCH="arm64-apple-darwin"
-
-          wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
-          tar -xzf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
-          sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin
-          rm -rf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz bitcoin-${BITCOIND_VERSION}
+        uses: actions/checkout@v2.0.0
 
       - name: Install dependencies
         run: |
-          export PATH="/usr/local/opt:/Users/runner/.local/bin:/opt/homebrew/bin/python3.10/bin:$PATH"
+          export PATH="/usr/local/opt:/Users/runner/.local/bin:/Users/runner/Library/Python/3.10/bin:$PATH"
+          export BITCOIN_VERSION=0.20.1
+          brew install wget python autoconf automake libtool python3 gmp gnu-sed gettext libsodium
 
-          brew install gnu-sed python@3.10 autoconf automake libtool protobuf
-          python3.10 -m pip install -U --user poetry==1.8.0 wheel pip mako
-          python3.10 -m poetry install
+          (
+            cd /tmp/
+            wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-osx64.tar.gz -O bitcoin.tar.gz
+            tar -xvzf bitcoin.tar.gz
+            sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin
+          )
 
-      - name: Build and install CLN
+          pip3 install --user poetry
+          poetry config virtualenvs.create false --local
+          poetry install
+          
+          ln -s /usr/local/Cellar/gettext/0.20.1/bin/xgettext /usr/local/opt
+          export PATH="/usr/local/opt:$PATH"
+
+      - name: Build
+        env:
+          VALGRIND: ${{ matrix.VALGRIND }}
+          DEVELOPER: ${{ matrix.DEVELOPER }}
+          EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }}
+          COMPILER: ${{ matrix.COMPILER }}
+          COMPAT: ${{ matrix.COMPAT }}
+          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
+          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
+          NO_PYTHON: ${{ matrix.NO_PYTHON }}
+          COPTFLAGS: ${{ matrix.COPTFLAGS }}
+          NETWORK: ${{ matrix.NETWORK }}
+          TEST_CMD: ${{ matrix.TEST_CMD }}
+          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
+          TEST_GROUP: ${{ matrix.TEST_GROUP }}
         run: |
-          export CPATH=/opt/homebrew/include
-          export LIBRARY_PATH=/opt/homebrew/lib
+          export PATH="/usr/local/opt:/Users/runner/.local/bin:/Users/runner/Library/Python/3.10/bin:$PATH"
+          export LDFLAGS="-L/usr/local/opt/sqlite/lib"
+          export CPPFLAGS="-I/usr/local/opt/sqlite/include"
 
-          python3.10 -m poetry run ./configure --disable-valgrind --disable-compat
-          python3.10 -m poetry run make
+          cat << EOF > pytest.ini
+          [pytest]
+          addopts=-p no:logging --color=yes --timeout=600 --timeout-method=thread --test-group-random-seed=42 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+          markers =
+              slow_test: marks tests as slow (deselect with '-m "not slow_test"')
+          EOF
 
-      - name: Start bitcoind in regtest mode
-        run: |
-          bitcoind -regtest -daemon
-          sleep 5
-
-      - name: Generate initial block
-        run: |
-          bitcoin-cli -regtest createwallet default_wallet
-          bitcoin-cli -regtest generatetoaddress 1 $(bitcoin-cli -regtest getnewaddress)
-          sleep 2
-
-      - name: Start CLN in regtest mode
-        run: |
-          lightningd/lightningd --network=regtest --log-file=/tmp/l1.log --daemon
-          sleep 5
-
-      - name: Verify CLN is running
-        run: |
-          cli/lightning-cli --regtest getinfo
+          ./configure
+          make

.github/workflows/pypi.yml

@@ -1,23 +1,19 @@
+---
 name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
-
 on:
+  # Only deploy if we're the result of a PR being merged
   push:
+    branches:
+      - master
     tags:
-      - 'v[0-9]+.[0-9]+'
+      # Semantic versioning tags
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      dist-location:
-        description: 'Distribution location (test/prod)'
-        default: 'test'
-        required: false
-
+      # Date style tags
+      - 'v[0-9]{2}.[0-9]{2}'
 jobs:
   deploy:
     name: Build and publish ${{ matrix.package }} 🐍
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: true
       matrix:
@@ -28,74 +24,61 @@ jobs:
             WORKDIR: contrib/pyln-testing
           - PACKAGE: pyln-proto
             WORKDIR: contrib/pyln-proto
+          # Bolt packages are handled differently
+          #- PACKAGE: pyn-bolt1
+          #  WORKDIR: contrib/pyln-spec/bolt1/
+          #- PACKAGE: pyn-bolt2
+          #  WORKDIR: contrib/pyln-spec/bolt2/
+          #- PACKAGE: pyn-bolt4
+          #  WORKDIR: contrib/pyln-spec/bolt4/
+          #- PACKAGE: pyn-bolt7
+          #  WORKDIR: contrib/pyln-spec/bolt7/
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+    - uses: actions/checkout@master
       with:
         # Need to fetch entire history in order to locate the version tag
         fetch-depth: 0
 
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+
+    - name: Install pypa/build and poetry
+      run: >-
+        python -m pip install build poetry --user
+
     - name: Check version tag
       run: >-
-        git describe --tags --always --dirty=-modded --abbrev=7
+        git describe --always --dirty=-modded --abbrev=7
 
-    - name: Setup Version
+    - name: Build a binary wheel and a source tarball
       env:
         WORKDIR: ${{ matrix.WORKDIR }}
       run: |
-        echo "VERSION=$(git describe --tags --abbrev=0).post$(git describe --tags --abbrev=1 | awk -F "-" '{print $2}')" >> $GITHUB_ENV
-
-    - name: Set up values
-      id: set-values
-      run: |
-        if [[ "${{ github.event.inputs.dist-location }}" != "" ]]; then
-          DISTLOCATION=${{ github.event.inputs.dist-location }}
-        elif [[ "${{ github.ref_type }}" == "tag" ]] && [[ ! "${{ github.ref_name }}" =~ rc ]]; then
-          DISTLOCATION="prod"
-        else
-          DISTLOCATION="test"
-        fi
-        echo "DISTLOCATION=$DISTLOCATION" >> $GITHUB_OUTPUT
-        echo "EVENT DISTLOCATION: ${{ github.event.inputs.dist-location }}"
-        echo "DISTRIBUTION LOCATION: $DISTLOCATION"
-  
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.10'
-
-    - name: Install Poetry
-      run: |
-        curl -sSL https://install.python-poetry.org | python3 -
-        echo "$HOME/.local/bin" >> $GITHUB_PATH
-        echo "PATH=$HOME/.local/bin:$PATH"
+        export VERSION=$(git describe --abbrev=0).post$(git describe --abbrev=1 | awk -F "-" '{print $2}')
+        cd ${{ env.WORKDIR}}
+        make upgrade-version NEW_VERSION=$VERSION
+        poetry build
 
     - name: Publish distribution 📦 to Test PyPI
-      if: github.repository == 'ElementsProject/lightning' && steps.set-values.outputs.DISTLOCATION == 'test'
+      if: github.repository == 'ElementsProject/lightning'
       env:
         POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.TEST_PYPI_API_TOKEN }}
         WORKDIR: ${{ matrix.WORKDIR }}
       run: |
-        echo "POETRY VERSION TEST: $(poetry --version)"
-        echo "Pyln VERSION: $VERSION"
-        cd ${{ env.WORKDIR }}
-        python3 -m pip config set global.timeout 150
+        cd ${{ env.WORKDIR}}
         poetry config repositories.testpypi https://test.pypi.org/legacy/
-        make upgrade-version NEW_VERSION=$VERSION
-        poetry build  --no-interaction
-        poetry publish --repository testpypi --no-interaction --skip-existing
+        poetry publish --repository testpypi --no-interaction
 
     - name: Publish distribution 📦 to PyPI
-      if: github.repository == 'ElementsProject/lightning' && steps.set-values.outputs.DISTLOCATION == 'prod'
+      if: startsWith(github.ref, 'refs/tags') && github.repository == 'ElementsProject/lightning'
       env:
         POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
         WORKDIR: ${{ matrix.WORKDIR }}
       run: |
-        echo "POETRY VERSION PUBLISH: $(poetry --version)"
-        cd ${{ env.WORKDIR }}
-        export VERSION=$(git describe --tags --abbrev=0)
-        echo "Pyln VERSION: $VERSION"
+        cd ${{ env.WORKDIR}}
+        export VERSION=$(git describe --abbrev=0)
         make upgrade-version NEW_VERSION=$VERSION
-        python3 -m pip config set global.timeout 150
-        poetry build  --no-interaction
-        poetry publish --no-interaction
+        poetry config repositories.testpypi https://test.pypi.org/legacy/
+        poetry publish --repository testpypi --no-interaction

.github/workflows/rdme-docs-sync.yml

@@ -1,51 +0,0 @@
-name: ReadMe Sync
-
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'doc/**'
-  workflow_dispatch:
-
-jobs:
-  rdme-docs-sync:  
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Check out repo 📚
-        uses: actions/checkout@v4
-
-      - name: Sync doc/getting-started/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/getting-started --key=${{ env.README_API_KEY }} --version=1
-            
-      - name: Sync doc/beginners-guide/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/beginners-guide --key=${{ env.README_API_KEY }} --version=1
-      
-      - name: Sync doc/node-operators-guide/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/node-operators-guide --key=${{ env.README_API_KEY }} --version=1
-      
-      - name: Sync doc/developers-guide/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/developers-guide --key=${{ env.README_API_KEY }} --version=1
-      
-      - name: Sync doc/contributing-to-core-lightning/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/contribute-to-core-lightning --key=${{ env.README_API_KEY }} --version=1

.github/workflows/readme-rpc-sync.yml

@@ -1,43 +0,0 @@
-name: ReadMe RPC Sync
-
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'doc/schemas/*.json'
-      - 'doc/*.1.md'
-      - 'doc/*.5.md'
-      - 'doc/*.8.md'
-      - 'doc/lightningd-*.7.md'
-      - 'doc/reckless.7.md'
-  workflow_dispatch:
-
-jobs:
-  rdme-rpc-sync:
-    runs-on: ubuntu-22.04
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
-
-    - name: Install python modules
-      run: |
-        python -m pip install requests mako grpcio-tools
-
-    - name: Install dependencies
-      run: bash -x .github/scripts/setup.sh
-
-    - name: Build (including rpc .md files)
-      run: |
-        ./configure --enable-debugbuild
-        make -j $(nproc)
-
-    - name: Set environment variable and run
-      env:
-        README_API_KEY: ${{ secrets.README_API_KEY }}
-      run: python .github/scripts/sync-rpc-cmds.py

.github/workflows/release.yml

@@ -1,191 +0,0 @@
----
-# https://docs.corelightning.org/docs/release-checklist
-name: "Release 🚀"
-on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      version:
-        description: 'Release version'
-        required: true
-      create_release:
-        description: Create a draft release
-        default: no
-        type: choice
-        options:
-          - yes
-          - no
-
-jobs:
-  check:
-    name: Check
-    outputs:
-      version: ${{ steps.capture.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.ref }}
-          fetch-depth: 0
-          fetch-tags: true
-
-      - name: Determine version
-        run: |
-          if [[ "${{ github.event.inputs.version }}" != "" ]]; then
-            VERSION="${{ github.event.inputs.version }}"
-          elif [ "${{ github.ref_type }}" == "tag" ]; then
-            VERSION="${{ github.ref_name }}"
-          else
-            echo "No release version provided and no tag found."
-            exit 1
-          fi
-          echo "VERSION=$VERSION" >> "$GITHUB_ENV"
-          echo "Determined version: $VERSION"
-
-      - name: Validate release
-        run: tools/check-release.sh --version=${VERSION}
-
-      - name: Catpure version output
-        id: capture
-        run: echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
-
-  releases:
-    name: Releases
-    needs: check
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false    # Let each build finish.
-      matrix:
-        target:
-          - 'bin-Fedora'
-          - 'bin-Ubuntu-focal'
-          - 'bin-Ubuntu-jammy'
-          - 'bin-Ubuntu-noble'
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-tags: true
-
-      # tools/build-release.sh requires lowdown
-      - name: Prepare base environment
-        run: |
-          sudo apt-get install -y lowdown
-          ./configure
-
-      - name: Build environment setup
-        run: |
-          distribution=$(echo ${{ matrix.target }} | cut -d'-' -f3)
-          echo "Building base image for ${distribution}"
-          sudo docker run --rm -v $(pwd):/build ubuntu:${distribution} bash -c "\
-              apt-get update && \
-              apt-get install -y debootstrap && \
-              debootstrap ${distribution} /build/${distribution}"
-          sudo tar -C ${distribution} -c . | docker import - ${distribution}
-
-          # Build Docker image
-          docker build -t cl-repro-${distribution} - < contrib/reprobuild/Dockerfile.${distribution}
-        if: contains(matrix.target, 'Ubuntu')
-
-      - name: Build release
-        run: tools/build-release.sh ${{ matrix.target }}
-
-      - name: Upload target artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          path: release/
-          name: ${{ matrix.target }}
-          if-no-files-found: error
-
-  artifact:
-    name: Construct release artifact
-    needs:
-      - check
-      - releases
-    env:
-      version: ${{ needs.check.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Merge artifacts
-        uses: actions/upload-artifact/merge@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          pattern: bin-*
-          delete-merged: true
-
-  release:
-    name: Sign and prepare release draft
-    needs:
-      - check
-      - artifact
-    env:
-      version: ${{ needs.check.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-tags: true
-
-      - name: Download artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          path: release/
-
-      - name: Import GPG keys
-        id: gpg
-        uses: crazy-max/ghaction-import-gpg@v6
-        with:
-          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.GPG_PASSPHRASE }}
-          trust_level: 5
-
-      - name: Set default GPG key
-        run: echo "default-key ${{ steps.gpg.outputs.keyid }}" >> ~/.gnupg/gpg.conf
-
-      - name: Sign release
-        run: |
-          sudo apt-get install -y lowdown
-          ./configure
-          tools/build-release.sh --without-zip sign
-          mv release/SHA256SUMS.asc${{ steps.gpg.outputs.keyid  }} release/SHA256SUMS.asc
-
-      - name: Upload signed artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          overwrite: true
-          path: release/
-
-      - name: Determine release data
-        id: release_data
-        run: |
-          VERSION=${{ env.version }}
-          CHANGELOG_VERSION=${VERSION#v}
-          echo "CHANGELOG_VERSION=$CHANGELOG_VERSION"
-          echo "changelog_version=$CHANGELOG_VERSION" >> "$GITHUB_OUTPUT"
-
-          CHANGELOG_TITLE=$(grep "## \[${CHANGELOG_VERSION}\]" CHANGELOG.md)
-          echo "CHANGELOG_TITLE=$CHANGELOG_TITLE"
-          echo "changelog_title=$CHANGELOG_TITLE" >> "$GITHUB_OUTPUT"
-
-          RELEASE_TITLE=$(echo $CHANGELOG_TITLE | cut -d'"' -f2)
-          echo "RELEASE_TITLE=$RELEASE_TITLE"
-          echo "release_title=$RELEASE_TITLE" >> "$GITHUB_OUTPUT"
-
-      - name: Prepare release draft
-        if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.create_release == 'yes')
-        uses: softprops/action-gh-release@v2
-        with:
-          name: "${{ env.version }} ${{ steps.release_data.outputs.release_title }}"
-          tag_name: ${{ env.version }}
-          draft: true
-          prerelease: contains(env.version, "-rc")
-          files: release/*
-          fail_on_unmatched_files: true

.github/workflows/repro.yml

@@ -1,113 +0,0 @@
----
-# https://docs.corelightning.org/docs/repro
-name: Repro Build Nightly
-on:
-  # 05:00 Berlin, 03:00 UTC, 23:00 New York, 20:00 Los Angeles
-  schedule:
-    - cron: "0 3 * * *"
-  workflow_dispatch:
-
-jobs:
-  ubuntu:
-    name: "Ubuntu repro build: ${{ matrix.version }}"
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: false    # Let each build finish.
-      matrix:
-        version: ['focal', 'jammy', 'noble']
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-
-      - name: Build environment setup - ${{ matrix.version }}
-        run: |
-          echo "Building base image for ${{ matrix.version }}"
-          echo "STEP=Build environment setup" >> "$GITHUB_ENV"
-          sudo docker run --rm -v $(pwd):/build ubuntu:${{ matrix.version }} bash -c "\
-            apt-get update && \
-            apt-get install -y debootstrap && \
-            debootstrap ${{ matrix.version }} /build/${{ matrix.version }}"
-          sudo tar -C ${{ matrix.version }} -c . | docker import - ${{ matrix.version }}
-
-      - name: Builder image setup - ${{ matrix.version }}
-        run: |
-          echo "STEP=Builder image setup" >> "$GITHUB_ENV"
-          docker build -t cl-repro-${{ matrix.version }} - < contrib/reprobuild/Dockerfile.${{ matrix.version }}
-
-      - name: Build reproducible image and store Git state - ${{ matrix.version }}
-        run: |
-          echo "STEP=Build reproducible image and store Git state" >> "$GITHUB_ENV"
-
-          # Create release directory.
-          mkdir $GITHUB_WORKSPACE/release
-
-          # Perform the repro build.
-          docker run --name cl-build -v $GITHUB_WORKSPACE:/repo -e FORCE_MTIME=$(date +%F) -t cl-repro-${{ matrix.version }}
-
-          # Commit the image in order to inspect the build later.
-          docker commit cl-build cl-repro
-
-          # Inspect the version.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "make version > /repo/release/version.txt"
-
-          # Inspect the Git tree state.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "\
-              git --no-pager status > /repo/release/git.log && \
-              git --no-pager diff >> /repo/release/git.log"
-
-          # Change permissions on the release files for access by the runner environment.
-          sudo chown -R runner $GITHUB_WORKSPACE/release
-
-      - name: Assert clean version - ${{ matrix.version }}
-        run: |
-          echo "STEP=Assert clean version" >> "$GITHUB_ENV"
-          echo 'Version:'
-          cat release/version.txt
-          echo -e
-
-          releasefile=$(ls release/clightning-*)
-          echo 'Release file:'
-          ls -al release/clightning-*
-          echo -e
-          if [ -n "$(cat release/version.txt | sed -n '/-modded/p')" ] || \
-             [ -n "$(echo $releasefile | sed -n '/-modded/p')" ]; then
-            echo "Git Status and Diff:"
-            cat release/git.log
-            echo -e
-            echo 'Error: release modded / dirty tree.'
-            exit 1
-          else
-            echo 'Success! Clean release.'
-          fi
-
-      - name: Upload release artifact - ${{ matrix.version }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: release-${{ matrix.version }}
-          path: release
-          retention-days: 3 # Automatically delete after 3 days
-
-      - name: Send email on failure
-        if: ${{ failure() }}
-        uses: dawidd6/action-send-mail@v3
-        with:
-          server_address: smtp.gmail.com
-          server_port: 587
-          username: ${{ secrets.EMAIL_USERNAME }}
-          password: ${{ secrets.EMAIL_PASSWORD }}
-          from: ${{ secrets.EMAIL_USERNAME }}
-          to: ${{ vars.DISTRIBUTION_LIST }}
-          subject: "CI Failure: Step ${{ env.STEP }} failed for distro ${{ matrix.version }}"
-          convert_markdown: true
-          html_body: |
-            <html>
-              <body>
-                <p>GitHub Workflow ${{ github.workflow }} Failed! For more details, click on the action below.</p>
-                <strong>Failure Details:</strong><br/>
-                <strong>Event: </strong>${{ github.event_name }}<br/>
-                <strong>Job: </strong>${{ github.job }}<br/>
-                <strong>Distro: </strong>${{ matrix.version }}<br/>
-                <strong>Step: </strong>${{ env.STEP }}<br/>
-                <strong>Action: </strong><a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}</a><br/>
-              </body>
-            </html>

.gitignore

@@ -57,10 +57,7 @@ tests/plugins/test_selfdisable_after_getmanifest
 
 # Ignore generated files
 devtools/features
-doc/schemas/sql.json
-doc/*.7.md
-doc/*.[1578]
-doc/reckless*.[1578]
+doc/lightning*.[1578]
 *_sqlgen.[ch]
 *_wiregen.[ch]
 *_printgen.[ch]
@@ -69,36 +66,17 @@ tests/node_pb2.py
 tests/node_pb2_grpc.py
 tests/primitives_pb2.py
 tests/primitives_pb2_grpc.py
-tests/autogenerate-examples-status.log
-tests/autogenerate-examples.json
-tests/autogenerate-examples-repeat.log
 
 # Ignore unrelated stuff
 .DS_Store
 .gdb_history
-.python-version
-compile_commands.json
 
 # Rust targets
 target
 plugins/cln-grpc
-plugins/clnrest
 
 # Build directories
 bionic/
 focal/
 jammy/
-noble/
 release/
-.vscode/
-.cache/
-
-# Ignore release verification Sha256Sums
-SHA256SUMS-*
-doc/.doc_version
-autogenerate-examples-status.log
-
-# Ignore nix outputs
-result
-result-[0-9]*
-tests/plugins/channeld_fakenet

.gitlab-ci.yml

@@ -10,7 +10,7 @@ build:
       curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN"
       -X POST
       --data '{"state": "pending", "description": "Gitlab-CI is building the commit", "context": "gitlab-ci"}'
-      https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_COMMIT_SHA || true
+      https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_BUILD_REF || true
   script:
     - make
     - make -j 12 check
@@ -27,7 +27,7 @@ update-status-fail:
   when: on_failure
   script:
     - >-
-      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "failure", "description": "Gitlab-CI build failed, please contact @cdecker for details about build #$CI_JOB_ID.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_COMMIT_SHA || true
+      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "failure", "description": "Gitlab-CI build failed, please contact @cdecker for details about build #$CI_BUILD_ID.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_BUILD_REF || true
 
 update-status-success:
   image: tutum/curl
@@ -35,4 +35,4 @@ update-status-success:
   when: on_success
   script:
     - >-
-      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "success", "description": "Gitlab-CI build succeeded.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_COMMIT_SHA || true
+      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "success", "description": "Gitlab-CI build succeeded.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_BUILD_REF || true

.gitmodules

@@ -14,6 +14,10 @@
 [submodule "external/gheap"]
 	path = external/gheap
 	url = https://github.com/valyala/gheap
+[submodule "external/lnprototest"]
+	path = external/lnprototest
+	url = https://github.com/niftynei/lnprototest.git
+	branch = nifty/ripemd160-fallback
 [submodule "external/lowdown"]
 	path = external/lowdown
 	url = https://github.com/kristapsdz/lowdown.git

.pre-commit-config.yaml

@@ -1,13 +0,0 @@
-repos:
-- repo: https://github.com/astral-sh/ruff-pre-commit
-  # Ruff version.
-  rev: v0.8.0
-  hooks:
-    # Run the linter.
-    - id: ruff
-      args: [ --diff ]
-      exclude: "contrib/pyln-grpc-proto/pyln/grpc/(primitives|node)_pb2(|_grpc).py"
-    # Run the formatter.
-    - id: ruff-format
-      args: [ --diff ]
-      exclude: "contrib/pyln-grpc-proto/pyln/grpc/(primitives|node)_pb2(|_grpc).py"

.version

@@ -1 +0,0 @@
-25.05

Cargo.toml

@@ -2,12 +2,9 @@
 strip = "debuginfo"
 
 [workspace]
-resolver = "2"
 members = [
-    "cln-rpc",
-    "cln-grpc",
-    "plugins",
-    "plugins/grpc-plugin",
-    "plugins/rest-plugin",
-    "plugins/lsps-plugin",
+  "cln-rpc",
+  "cln-grpc",
+  "plugins",
+  "plugins/grpc-plugin",
 ]

Dockerfile

@@ -1,35 +1,11 @@
-# This Dockerfile is used by buildx to build ARM64, AMD64, and ARM32 Docker images from an AMD64 host.
-# To speed up the build process, we are cross-compiling rather than relying on QEMU.
-# There are four main stages:
-# * downloader: Downloads specific binaries needed for core lightning for each architecture.
-# * builder: Cross-compiles for each architecture.
-# * builder-python: Builds Python dependencies for wss-proxy with QEMU.
-# * final: Creates the runtime image.
-
-ARG DEFAULT_TARGETPLATFORM="linux/amd64"
-ARG BASE_DISTRO="debian:bookworm-slim"
-
-FROM --platform=$BUILDPLATFORM ${BASE_DISTRO} AS base-downloader
-RUN set -ex \
-	&& apt-get update \
-	&& apt-get install -qq --no-install-recommends ca-certificates dirmngr wget qemu-user-static binfmt-support
-
-FROM base-downloader AS base-downloader-linux-amd64
-ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu
-ENV DESCHASHPLUGIN_ARCH=linux-amd64
-ENV DESCHASHPLUGIN_HASH=deadc00c68fac80b2718d92f69bf06acd8fff646228d497bbb76a4f0a12ca217
-
-FROM base-downloader AS base-downloader-linux-arm64
-ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu
-ENV DESCHASHPLUGIN_ARCH=linux-arm64
-ENV DESCHASHPLUGIN_HASH=d48c3e5aede77bd9cb72d78689ce12c0327f624435cb0496b3eacb92df416363
-
-FROM base-downloader AS base-downloader-linux-arm
-ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf
-ENV DESCHASHPLUGIN_ARCH=linux-arm
-ENV DESCHASHPLUGIN_HASH=f7df336c72dd1674bd18ff23862a410b6a9691a3e13752264dcffa0950e21c74
-
-FROM base-downloader-${TARGETOS}-${TARGETARCH} AS downloader
+# This dockerfile is meant to compile a core-lightning x64 image
+# It is using multi stage build:
+# * downloader: Download litecoin/bitcoin and qemu binaries needed for core-lightning
+# * builder: Compile core-lightning dependencies, then core-lightning itself with static linking
+# * final: Copy the binaries required at runtime
+# The resulting image uploaded to dockerhub will only contain what is needed for runtime.
+# From the root of the repository, run "docker build -t yourimage:yourtag ."
+FROM debian:bullseye-slim as downloader
 
 RUN set -ex \
 	&& apt-get update \
@@ -37,9 +13,12 @@ RUN set -ex \
 
 WORKDIR /opt
 
+RUN wget -qO /opt/tini "https://github.com/krallin/tini/releases/download/v0.18.0/tini" \
+    && echo "12d20136605531b09a2c2dac02ccee85e1b874eb322ef6baf7561cd93f93c855 /opt/tini" | sha256sum -c - \
+    && chmod +x /opt/tini
 
-ENV BITCOIN_VERSION=27.1
-ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
+ARG BITCOIN_VERSION=22.0
+ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
 ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/$BITCOIN_TARBALL
 ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS
 
@@ -49,292 +28,127 @@ RUN mkdir /opt/bitcoin && cd /opt/bitcoin \
     && grep $BITCOIN_TARBALL bitcoin | tee SHA256SUMS \
     && sha256sum -c SHA256SUMS \
     && BD=bitcoin-$BITCOIN_VERSION/bin \
-    && tar -xzvf $BITCOIN_TARBALL $BD/ --strip-components=1 \
+    && tar -xzvf $BITCOIN_TARBALL $BD/bitcoin-cli --strip-components=1 \
     && rm $BITCOIN_TARBALL
 
 ENV LITECOIN_VERSION 0.16.3
-ENV LITECOIN_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
+ENV LITECOIN_PGP_KEY FE3348877809386C
+ENV LITECOIN_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-x86_64-linux-gnu.tar.gz
+ENV LITECOIN_ASC_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-linux-signatures.asc
+ENV LITECOIN_SHA256 686d99d1746528648c2c54a1363d046436fd172beadaceea80bdc93043805994
 
 # install litecoin binaries
 RUN mkdir /opt/litecoin && cd /opt/litecoin \
     && wget -qO litecoin.tar.gz "$LITECOIN_URL" \
-    && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \
+    && echo "$LITECOIN_SHA256  litecoin.tar.gz" | sha256sum -c - \
+    && BD=litecoin-$LITECOIN_VERSION/bin \
+    && tar -xzvf litecoin.tar.gz $BD/litecoin-cli --strip-components=1 --exclude=*-qt \
     && rm litecoin.tar.gz
 
-ENV DESCHASHPLUGIN_URL https://github.com/nbd-wtf/invoicewithdescriptionhash/releases/download/v1.4/invoicewithdescriptionhash-v1.4-${DESCHASHPLUGIN_ARCH}.tar.gz
-ENV DESCHASHPLUGIN_SHA256 ${DESCHASHPLUGIN_HASH}
+ENV DESCHASHPLUGIN_URL https://github.com/fiatjaf/sparko/releases/download/invoicewithdescriptionhash-v1.2/invoicewithdescriptionhash_linux_amd64
+ENV DESCHASHPLUGIN_SHA256 E3EA0D076A26D774BA68D1D5E3FE48D267CE02D077933EF3CBAE1FC39007FB11
 RUN mkdir /opt/deschashplugin && cd /opt/deschashplugin \
-    && wget -qO invoicewithdescriptionhash.tar.gz "$DESCHASHPLUGIN_URL" \
-    && echo "$DESCHASHPLUGIN_SHA256  invoicewithdescriptionhash.tar.gz" | sha256sum -c - \
-    && tar -xzvf invoicewithdescriptionhash.tar.gz && rm invoicewithdescriptionhash.tar.gz \
+    && wget -qO invoicewithdescriptionhash "$DESCHASHPLUGIN_URL" \
+    && echo "$DESCHASHPLUGIN_SHA256  invoicewithdescriptionhash" | sha256sum -c - \
     && chmod a+x invoicewithdescriptionhash
 
-FROM --platform=${DEFAULT_TARGETPLATFORM} ${BASE_DISTRO} AS base-builder
+FROM debian:bullseye-slim as builder
+
+ENV LIGHTNINGD_VERSION=master
 RUN apt-get update -qq && \
     apt-get install -qq -y --no-install-recommends \
         autoconf \
         automake \
-        bison \
         build-essential \
         ca-certificates \
         curl \
         dirmngr \
-        flex \
         gettext \
         git \
         gnupg \
-        jq \
-        libicu-dev \
+        libpq-dev \
         libtool \
         libffi-dev \
-        pkg-config \
-        libssl-dev \
-        protobuf-compiler \
         python3 \
         python3-dev \
         python3-mako \
         python3-pip \
         python3-venv \
         python3-setuptools \
-        libev-dev \
-        libevent-dev \
-        qemu-user-static \
-        wget \
-        unzip \
-        tclsh
+        wget
 
-ENV PATH="/root/.local/bin:$PATH" \
-    PYTHON_VERSION=3 \
-    POETRY_VERSION=2.0.1
-RUN curl -sSL https://install.python-poetry.org | python3 - && \
-    poetry self add poetry-plugin-export
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install --upgrade pip setuptools wheel
+RUN wget -q https://zlib.net/zlib-1.2.12.tar.gz \
+&& tar xvf zlib-1.2.12.tar.gz \
+&& cd zlib-1.2.12 \
+&& ./configure \
+&& make \
+&& make install && cd .. && rm zlib-1.2.12.tar.gz && rm -rf zlib-1.2.12
 
-RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz -O zlib.tar.gz && \
-    wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip -O sqlite.zip && \
-    wget -q https://ftp.postgresql.org/pub/source/v17.1/postgresql-17.1.tar.gz -O postgres.tar.gz
+RUN apt-get install -y --no-install-recommends unzip tclsh \
+&& wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip \
+&& unzip sqlite-src-3290000.zip \
+&& cd sqlite-src-3290000 \
+&& ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension \
+&& make \
+&& make install && cd .. && rm sqlite-src-3290000.zip && rm -rf sqlite-src-3290000
+
+RUN wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz \
+&& tar xvf gmp-6.1.2.tar.xz \
+&& cd gmp-6.1.2 \
+&& ./configure --disable-assembly \
+&& make \
+&& make install && cd .. && rm gmp-6.1.2.tar.xz && rm -rf gmp-6.1.2
+
+ENV RUST_PROFILE=release
+ENV PATH=$PATH:/root/.cargo/bin/
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+RUN rustup toolchain install stable --component rustfmt --allow-downgrade
 
 WORKDIR /opt/lightningd
 COPY . /tmp/lightning
 RUN git clone --recursive /tmp/lightning . && \
     git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD)
-
-# Do not build python plugins (wss-proxy) here, python doesn't support cross compilation.
-RUN sed -i '/^wss-proxy/d' pyproject.toml && \
-    poetry lock && \
-    poetry export -o requirements.txt --without-hashes
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install -r requirements.txt && \
-    pip3 cache purge
-WORKDIR /
-
-FROM base-builder AS base-builder-linux-amd64
-
-ENV POSTGRES_CONFIG="--without-readline" \
-    PG_CONFIG=/usr/local/pgsql/bin/pg_config
-
-FROM base-builder AS base-builder-linux-arm64
-ENV target_host=aarch64-linux-gnu \
-    target_host_rust=aarch64-unknown-linux-gnu \
-    target_host_qemu=qemu-aarch64-static
-
-RUN apt-get install -qq -y --no-install-recommends \
-        libc6-arm64-cross \
-        gcc-${target_host} \
-        g++-${target_host}
-
-ENV AR=${target_host}-ar \
-    AS=${target_host}-as \
-    CC=${target_host}-gcc \
-    CXX=${target_host}-g++ \
-    LD=${target_host}-ld \
-    STRIP=${target_host}-strip \
-    QEMU_LD_PREFIX=/usr/${target_host} \
-    HOST=${target_host} \
-    TARGET=${target_host_rust} \
-    RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
-    PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
-
-ENV ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
-    SQLITE_CONFIG="--host=${target_host} --prefix=${QEMU_LD_PREFIX}" \
-    POSTGRES_CONFIG="--without-readline --prefix=${QEMU_LD_PREFIX}" \
-    PG_CONFIG="${QEMU_LD_PREFIX}/bin/pg_config"
-
-FROM base-builder AS base-builder-linux-arm
-
-ENV target_host=arm-linux-gnueabihf \
-    target_host_rust=armv7-unknown-linux-gnueabihf \
-    target_host_qemu=qemu-arm-static
-
-RUN apt-get install -qq -y --no-install-recommends \
-        libc6-armhf-cross \
-        gcc-${target_host} \
-        g++-${target_host}
-
-ENV AR=${target_host}-ar \
-    AS=${target_host}-as \
-    CC=${target_host}-gcc \
-    CXX=${target_host}-g++ \
-    LD=${target_host}-ld \
-    STRIP=${target_host}-strip \
-    QEMU_LD_PREFIX=/usr/${target_host} \
-    HOST=${target_host} \
-    TARGET=${target_host_rust} \
-    RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
-    PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
-
-ENV ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
-    SQLITE_CONFIG="--host=${target_host} --prefix=${QEMU_LD_PREFIX}" \
-    POSTGRES_CONFIG="--without-readline --prefix=${QEMU_LD_PREFIX}" \
-    PG_CONFIG="${QEMU_LD_PREFIX}/bin/pg_config"
-
-FROM base-builder-${TARGETOS}-${TARGETARCH} AS builder
-
-ENV LIGHTNINGD_VERSION=master
-
-RUN mkdir zlib && tar xvf zlib.tar.gz -C zlib --strip-components=1 \
-    && cd zlib \
-    && ./configure ${ZLIB_CONFIG} \
-    && make \
-    && make install && cd .. && \
-    rm zlib.tar.gz && \
-    rm -rf zlib
-
-RUN unzip sqlite.zip \
-    && cd sqlite-* \
-    && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension ${SQLITE_CONFIG} \
-    && make \
-    && make install && cd .. && rm sqlite.zip && rm -rf sqlite-*
-
-RUN mkdir postgres && tar xvf postgres.tar.gz -C postgres --strip-components=1 \
-    && cd postgres \
-    && ./configure ${POSTGRES_CONFIG} \
-    && cd src/include \
-    && make install \
-    && cd ../interfaces/libpq \
-    && make install \
-    && cd ../../bin/pg_config \
-    && make install \
-    && cd ../../../../ && \
-    rm postgres.tar.gz && \
-    rm -rf postgres && \
-    ldconfig "$(${PG_CONFIG} --libdir)"
-
-# Save libpq to a specific location to copy it into the final image.
-RUN mkdir /var/libpq && cp -a "$(${PG_CONFIG} --libdir)"/libpq.* /var/libpq
-
-ENV RUST_PROFILE=release \
-    PATH="/root/.cargo/bin:/root/.local/bin:$PATH"
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
-RUN rustup toolchain install stable --component rustfmt --allow-downgrade
-
-COPY --from=downloader /usr/bin/${target_host_qemu} /usr/bin/${target_host_qemu}
-WORKDIR /opt/lightningd
-
-# If cross-compiling, need to tell it to cargo.
-RUN ( ! [ -n "${target_host}" ] ) || \
-    (mkdir -p .cargo && echo "[target.${target_host_rust}]\nlinker = \"${target_host}-gcc\"" > .cargo/config)
-
-# Weird errors with cargo for cln-grpc on arm7 https://github.com/ElementsProject/lightning/issues/6596
-RUN ( ! [ "${target_host}" = "arm-linux-gnueabihf" ] ) || \
-    (sed -i '/documentation = "https:\/\/docs.rs\/cln-grpc"/a include = ["**\/*.*"]' cln-grpc/Cargo.toml)
-
-# Ensure that the desired grpcio-tools & protobuf versions are installed
-# https://github.com/ElementsProject/lightning/pull/7376#issuecomment-2161102381
-RUN poetry lock && poetry install && \
-    poetry self add poetry-plugin-export
-
-# Ensure that git differences are removed before making bineries, to avoid `-modded` suffix
-# poetry.lock changed due to pyln-client, pyln-proto and pyln-testing version updates
-# pyproject.toml was updated to exclude wss-proxy plugins in base-builder stage
-RUN git reset --hard HEAD
-
-RUN ./configure --prefix=/tmp/lightning_install --enable-static && poetry run make install
-
-# Export the requirements for the plugins so we can install them in builder-python stage
-WORKDIR /opt/lightningd/plugins/wss-proxy
-RUN poetry lock && poetry export -o requirements.txt --without-hashes
-WORKDIR /opt/lightningd
-RUN echo 'RUSTUP_INSTALL_OPTS="${RUSTUP_INSTALL_OPTS}"' > /tmp/rustup_install_opts.txt
-
-# We need to build python plugins on the target's arch because python doesn't support cross build
-FROM ${BASE_DISTRO} AS builder-python
-RUN apt-get update -qq && \
-    apt-get install -qq -y --no-install-recommends \
-        git \
-        curl \
-        libtool \
-        pkg-config \
-        autoconf \
-        automake \
-        build-essential \
-        libffi-dev \
-        libssl-dev \
-        python3 \
-        python3-dev \
-        python3-pip \
-        python3-venv && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
+ARG DEVELOPER=0
 ENV PYTHON_VERSION=3
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install --upgrade pip setuptools wheel
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python3 - \
+    && pip3 install -U pip \
+    && pip3 install -U wheel \
+    && /root/.local/bin/poetry config virtualenvs.create false \
+    && /root/.local/bin/poetry install
+RUN pip3 install mrkd
+RUN ./configure --prefix=/tmp/lightning_install --enable-static && make -j3 DEVELOPER=${DEVELOPER} && make install
 
-# Copy rustup_install_opts.txt file from builder
-COPY --from=builder /tmp/rustup_install_opts.txt /tmp/rustup_install_opts.txt
-# Setup ENV $RUSTUP_INSTALL_OPTS for this stage
-RUN export $(cat /tmp/rustup_install_opts.txt)
-ENV PATH="/root/.cargo/bin:/root/.venvs/cln/bin:$PATH"
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
+FROM debian:bullseye-slim as final
 
-WORKDIR /opt/lightningd/plugins/wss-proxy
-COPY --from=builder /opt/lightningd/plugins/wss-proxy/requirements.txt .
-RUN pip3 install -r requirements.txt
-RUN pip3 cache purge
+ARG TRACE_TOOLS=false
+ENV TRACE_TOOLS=$TRACE_TOOLS
+ENV TRACE_LOCATION=/opt/traces
+VOLUME /opt/traces
 
-WORKDIR /opt/lightningd
+COPY --from=downloader /opt/tini /usr/bin/tini
+RUN apt-get update && apt-get install -y --no-install-recommends socat inotify-tools python3 python3-pip libpq5\
+    && \
+    ( ! $TRACE_TOOLS || \
+        ( \
+            apt-get install -y --no-install-recommends perl linux-base curl ca-certificates && \
+            mkdir FlameGraph && cd FlameGraph && \
+            curl -Lo FlameGraph.tar.gz "https://github.com/brendangregg/FlameGraph/archive/v1.0.tar.gz" && \
+            tar -zxvf FlameGraph.tar.gz --strip-components=1 && rm FlameGraph.tar.gz && cd .. \
+        ) \
+    ) \
+    && rm -rf /var/lib/apt/lists/*
 
-FROM ${BASE_DISTRO} AS final
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-      tini \
-      socat \
-      inotify-tools \
-      jq \
-      python3 \
-      python3-pip && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-ENV LIGHTNINGD_DATA=/root/.lightning \
-    LIGHTNINGD_RPC_PORT=9835 \
-    LIGHTNINGD_PORT=9735 \
-    LIGHTNINGD_NETWORK=bitcoin
+ENV LIGHTNINGD_DATA=/root/.lightning
+ENV LIGHTNINGD_RPC_PORT=9835
+ENV LIGHTNINGD_PORT=9735
+ENV LIGHTNINGD_NETWORK=bitcoin
 
 RUN mkdir $LIGHTNINGD_DATA && \
     mkdir /etc/bundledplugins && \
     mkdir $LIGHTNINGD_DATA/plugins && \
     touch $LIGHTNINGD_DATA/config
 VOLUME [ "/root/.lightning" ]
-
-# Take libpq directly from builder.
-RUN mkdir /var/libpq && mkdir -p /usr/local/pgsql/lib
-RUN --mount=type=bind,from=builder,source=/var/libpq,target=/var/libpq,rw \
-    cp -a /var/libpq/libpq.* /usr/local/pgsql/lib && \
-    echo "/usr/local/pgsql/lib" > /etc/ld.so.conf.d/libpq.conf && \
-    ldconfig
-
 COPY --from=builder /tmp/lightning_install/ /usr/local/
-COPY --from=builder-python /root/.venvs/cln/lib/python3.11/site-packages /usr/local/lib/python3.11/dist-packages/
 COPY --from=downloader /opt/bitcoin/bin /usr/bin
 COPY --from=downloader /opt/litecoin/bin /usr/bin
 COPY --from=downloader /opt/deschashplugin $LIGHTNINGD_DATA/plugins

LICENSE

@@ -1,7 +1,7 @@
 Note: the modules in the ccan/ directory have their own licenses, but
 the rest of the code is covered by the following (BSD-MIT) license:
 
-	Copyright Rusty Russell (Blockstream) 2015-2024.
+	Copyright Rusty Russell (Blockstream) 2015-2022.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,10 +1,7 @@
 #! /usr/bin/make
 
 # Extract version from git, or if we're from a zipfile, use dirname
-VERSION=$(shell git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|v\1|gp')
-
-# Next release.
-CLN_NEXT_VERSION := v25.05
+VERSION=$(shell git describe --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|\1|gp')
 
 # --quiet / -s means quiet, dammit!
 ifeq ($(findstring s,$(word 1, $(MAKEFLAGS))),s)
@@ -26,7 +23,7 @@ CCANDIR := ccan
 
 # Where we keep the BOLT RFCs
 BOLTDIR := ../bolts/
-DEFAULT_BOLTVERSION := ccfa38ed4f592c3711156bb4ded77f44ec01101d
+DEFAULT_BOLTVERSION := f32c6ddb5f11b431c9bb4f501cdec604172a90de
 # Can be overridden on cmdline.
 BOLTVERSION := $(DEFAULT_BOLTVERSION)
 
@@ -36,7 +33,7 @@ SORT=LC_ALL=C sort
 
 
 ifeq ($V,1)
-VERBOSE = $(ECHO) '$(subst ','\'',$(2))'; $(2)
+VERBOSE = $(ECHO) '$(2)'; $(2)
 else
 VERBOSE = $(ECHO) $(1); $(2)
 endif
@@ -46,7 +43,21 @@ VG=VALGRIND=1 valgrind -q --error-exitcode=7
 VG_TEST_ARGS = --track-origins=yes --leak-check=full --show-reachable=yes --errors-for-leak-kinds=all
 endif
 
-ifeq ($(DEBUGBUILD),1)
+SANITIZER_FLAGS :=
+
+ifneq ($(ASAN),0)
+SANITIZER_FLAGS += -fsanitize=address
+endif
+
+ifneq ($(UBSAN),0)
+SANITIZER_FLAGS += -fsanitize=undefined
+endif
+
+ifneq ($(FUZZING), 0)
+SANITIZER_FLAGS += -fsanitize=fuzzer-no-link
+endif
+
+ifeq ($(DEVELOPER),1)
 DEV_CFLAGS=-DCCAN_TAKE_DEBUG=1 -DCCAN_TAL_DEBUG=1 -DCCAN_JSON_OUT_DEBUG=1
 else
 DEV_CFLAGS=
@@ -56,10 +67,6 @@ ifeq ($(COVERAGE),1)
 COVFLAGS = --coverage
 endif
 
-ifeq ($(CLANG_COVERAGE),1)
-COVFLAGS+=-fprofile-instr-generate -fcoverage-mapping
-endif
-
 ifeq ($(PIE),1)
 PIE_CFLAGS=-fPIE -fPIC
 PIE_LDFLAGS=-pie
@@ -72,14 +79,18 @@ endif
 
 # (method=thread to support xdist)
 PYTEST_OPTS := -v -p no:logging $(PYTEST_OPTS)
-MY_CHECK_PYTHONPATH=$${PYTHONPATH}$${PYTHONPATH:+:}$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7:$(shell pwd)/contrib/pyln-grpc-proto
+MY_CHECK_PYTHONPATH=$${PYTHONPATH}$${PYTHONPATH:+:}$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/external/lnprototest:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7
 # Collect generated python files to be excluded from lint checks
 PYTHON_GENERATED= \
-	contrib/pyln-grpc-proto/pyln/grpc/primitives_pb2.py \
-	contrib/pyln-grpc-proto/pyln/grpc/node_pb2_grpc.py \
-	contrib/pyln-grpc-proto/pyln/grpc/node_pb2.py \
+	contrib/pyln-testing/pyln/testing/primitives_pb2.py \
+	contrib/pyln-testing/pyln/testing/node_pb2_grpc.py \
+	contrib/pyln-testing/pyln/testing/node_pb2.py \
 	contrib/pyln-testing/pyln/testing/grpc2py.py
 
+# Options to pass to cppcheck. Mostly used to exclude files that are
+# generated with external tools that we don't have control over
+CPPCHECK_OPTS=-q --language=c --std=c11 --error-exitcode=1 --suppressions-list=.cppcheck-suppress --inline-suppr
+
 # This is where we add new features as bitcoin adds them.
 FEATURES :=
 
@@ -180,7 +191,6 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/json_out/json_out.h		\
 	$(CCANDIR)/ccan/likely/likely.h			\
 	$(CCANDIR)/ccan/list/list.h			\
-	$(CCANDIR)/ccan/lqueue/lqueue.h			\
 	$(CCANDIR)/ccan/mem/mem.h			\
 	$(CCANDIR)/ccan/membuf/membuf.h			\
 	$(CCANDIR)/ccan/noerr/noerr.h			\
@@ -228,24 +238,10 @@ ALL_TEST_PROGRAMS :=
 ALL_TEST_GEN :=
 ALL_FUZZ_TARGETS :=
 ALL_C_SOURCES :=
-ALL_C_HEADERS :=
+ALL_C_HEADERS := header_versions_gen.h version_gen.h
 # Extra (non C) targets that should be built by default.
 DEFAULT_TARGETS :=
 
-# Installation directories
-exec_prefix = $(PREFIX)
-bindir = $(exec_prefix)/bin
-libexecdir = $(exec_prefix)/libexec
-pkglibexecdir = $(libexecdir)/$(PKGNAME)
-plugindir = $(pkglibexecdir)/plugins
-datadir = $(PREFIX)/share
-docdir = $(datadir)/doc/$(PKGNAME)
-mandir = $(datadir)/man
-man1dir = $(mandir)/man1
-man5dir = $(mandir)/man5
-man7dir = $(mandir)/man7
-man8dir = $(mandir)/man8
-
 # M1 macos machines with homebrew will install the native libraries in
 # /opt/homebrew instead of /usr/local, most likely because they
 # emulate x86_64 compatibility via Rosetta, and wanting to keep the
@@ -254,16 +250,13 @@ man8dir = $(mandir)/man8
 ifeq ("$(OS)-$(ARCH)", "Darwin-arm64")
 CPATH := /opt/homebrew/include
 LIBRARY_PATH := /opt/homebrew/lib
-LDFLAGS := -L/opt/homebrew/opt/sqlite/lib
-CPPFLAGS := -I/opt/homebrew/opt/sqlite/include
-PKG_CONFIG_PATH=/opt/homebrew/opt/sqlite/lib/pkgconfig
 else
 CPATH := /usr/local/include
 LIBRARY_PATH := /usr/local/lib
 endif
 
-CPPFLAGS += -DCLN_NEXT_VERSION="\"$(CLN_NEXT_VERSION)\"" -DPKGLIBEXECDIR="\"$(pkglibexecdir)\"" -DBINDIR="\"$(bindir)\"" -DPLUGINDIR="\"$(plugindir)\"" -DCCAN_TAL_NEVER_RETURN_NULL=1
-CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I$(CPATH) $(SQLITE3_CFLAGS) $(SODIUM_CFLAGS) $(POSTGRES_INCLUDE) $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) $(CSANFLAGS)
+CPPFLAGS += -DBINTOPKGLIBEXECDIR="\"$(shell sh tools/rel.sh $(bindir) $(pkglibexecdir))\""
+CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I$(CPATH) $(SQLITE3_CFLAGS) $(POSTGRES_INCLUDE) $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) -DBUILD_ELEMENTS=1
 
 # If CFLAGS is already set in the environment of make (to whatever value, it
 # does not matter) then it would export it to subprocesses with the above value
@@ -275,15 +268,16 @@ unexport CFLAGS
 # We can get configurator to run a different compile cmd to cross-configure.
 CONFIGURATOR_CC := $(CC)
 
-LDFLAGS += $(PIE_LDFLAGS) $(CSANFLAGS) $(COPTFLAGS)
+LDFLAGS += $(PIE_LDFLAGS) $(SANITIZER_FLAGS) $(COPTFLAGS)
+CFLAGS += $(SANITIZER_FLAGS)
 
 ifeq ($(STATIC),1)
 # For MacOS, Jacob Rapoport <jacob@rumblemonkey.com> changed this to:
-#  -L/usr/local/lib -lsqlite3 -lz -Wl,-lm -lpthread -ldl $(COVFLAGS)
+#  -L/usr/local/lib -Wl,-lgmp -lsqlite3 -lz -Wl,-lm -lpthread -ldl $(COVFLAGS)
 # But that doesn't static link.
-LDLIBS = -L$(CPATH) -Wl,-dn $(SQLITE3_LDLIBS) -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
+LDLIBS = -L$(CPATH) -Wl,-dn -lgmp $(SQLITE3_LDLIBS) -lz -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
 else
-LDLIBS = -L$(CPATH) -lm $(SQLITE3_LDLIBS) $(COVFLAGS)
+LDLIBS = -L$(CPATH) -lm -lgmp $(SQLITE3_LDLIBS) -lz $(COVFLAGS)
 endif
 
 # If we have the postgres client library we need to link against it as well
@@ -291,25 +285,35 @@ ifeq ($(HAVE_POSTGRES),1)
 LDLIBS += $(POSTGRES_LDLIBS)
 endif
 
-default: show-flags gen all-programs all-test-programs doc-all default-targets $(PYTHON_GENERATED)
+default: show-flags all-programs all-test-programs doc-all default-targets
 
 ifneq ($(SUPPRESS_GENERATION),1)
 FORCE = FORCE
-FORCE:
+FORCE::
 endif
 
 show-flags: config.vars
 	@$(ECHO) "CC: $(CC) $(CFLAGS) -c -o"
 	@$(ECHO) "LD: $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) -o"
 
-# We will re-generate, but we won't generate for the first time!
-ccan/config.h config.vars &: configure ccan/tools/configurator/configurator.c
-	@if [ ! -f config.vars ]; then echo 'File config.vars not found: you must run ./configure before running make.' >&2; exit 1; fi
+ccan/config.h: config.vars configure ccan/tools/configurator/configurator.c
 	./configure --reconfigure
 
+config.vars:
+	@echo 'File config.vars not found: you must run ./configure before running make.' >&2
+	@exit 1
+
 %.o: %.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 
+# '_exp' inserted before _wiregen.[ch] to demark experimental
+# spec-derived headers, which are *not* committed into git.
+ifeq ($(EXPERIMENTAL_FEATURES),1)
+EXP := _exp
+else
+EXP :=
+endif
+
 # tools/update-mocks.sh does nasty recursive make, must not do this!
 ifeq ($(SUPPRESS_GENERATION),1)
 SHA256STAMP_CHANGED = false
@@ -366,45 +370,33 @@ include connectd/Makefile
 include lightningd/Makefile
 include cli/Makefile
 include doc/Makefile
-include contrib/msggen/Makefile
 include devtools/Makefile
 include tools/Makefile
-ifneq ($(RUST),0)
-include cln-rpc/Makefile
-include cln-grpc/Makefile
-endif
 include plugins/Makefile
 include tests/plugins/Makefile
 
 ifneq ($(FUZZING),0)
 	include tests/fuzz/Makefile
 endif
+ifneq ($(RUST),0)
+	include cln-rpc/Makefile
+	include cln-grpc/Makefile
 
-ifneq ($V,1)
-MSGGEN_ARGS := -s
-endif
-
-$(MSGGEN_GENALL)&: contrib/msggen/msggen/schema.json
-	@$(call VERBOSE, "msggen $@", PYTHONPATH=contrib/msggen $(PYTHON) contrib/msggen/msggen/__main__.py $(MSGGEN_ARGS) generate)
-
-# The compiler assumes that the proto files are in the same
-# directory structure as the generated files will be. Since we
-# don't do that we need to path the files up.
-GRPC_DIR = contrib/pyln-grpc-proto/pyln
-GRPC_PATH = $(GRPC_DIR)/grpc
-
-GRPC_GEN = \
-	$(GRPC_PATH)/node_pb2.py \
-	$(GRPC_PATH)/node_pb2_grpc.py \
-	$(GRPC_PATH)/primitives_pb2.py
+GRPC_GEN = contrib/pyln-testing/pyln/testing/node_pb2.py \
+	contrib/pyln-testing/pyln/testing/node_pb2_grpc.py \
+	contrib/pyln-testing/pyln/testing/primitives_pb2.py
 
 ALL_TEST_GEN += $(GRPC_GEN)
 
-$(GRPC_GEN) &: cln-grpc/proto/node.proto cln-grpc/proto/primitives.proto
-	$(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/node.proto --python_out=$(GRPC_PATH)/ --grpc_python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
-	$(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/primitives.proto --python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
-	find $(GRPC_DIR)/ -type f -name "*.py" -print0 | xargs -0 sed -i'.bak' -e 's/^import \(.*\)_pb2 as .*__pb2/from pyln.grpc import \1_pb2 as \1__pb2/g'
-	find $(GRPC_DIR)/ -type f -name "*.py.bak" -print0 | xargs -0 rm -f
+$(GRPC_GEN): cln-grpc/proto/node.proto cln-grpc/proto/primitives.proto
+	python -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/node.proto --python_out=contrib/pyln-testing/pyln/testing/ --grpc_python_out=contrib/pyln-testing/pyln/testing/ --experimental_allow_proto3_optional
+	python -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/primitives.proto --python_out=contrib/pyln-testing/pyln/testing/ --experimental_allow_proto3_optional
+	# The compiler assumes that the proto files are in the same
+	# directory structure as the generated files will be. Since we
+	# don't do that we need to path the files up.
+	find contrib/pyln-testing/pyln/testing/ -type f -name "*.py" -print0 | xargs -0 sed -i 's/^import \(.*\)_pb2 as .*__pb2/from . import \1_pb2 as \1__pb2/g'
+
+endif
 
 # We make pretty much everything depend on these.
 ALL_GEN_HEADERS := $(filter %gen.h,$(ALL_C_HEADERS))
@@ -419,8 +411,7 @@ ALL_NONGEN_SRCFILES := $(ALL_NONGEN_HEADERS) $(ALL_NONGEN_SOURCES)
 BIN_PROGRAMS = \
 	       cli/lightning-cli \
 	       lightningd/lightningd \
-	       tools/lightning-hsmtool\
-	       tools/reckless
+	       tools/lightning-hsmtool
 PKGLIBEXEC_PROGRAMS = \
 	       lightningd/lightning_channeld \
 	       lightningd/lightning_closingd \
@@ -432,18 +423,8 @@ PKGLIBEXEC_PROGRAMS = \
 	       lightningd/lightning_openingd \
 	       lightningd/lightning_websocketd
 
-mkdocs.yml: $(MANPAGES:=.md)
-	@$(call VERBOSE, "genidx $@", \
-	  find doc -maxdepth 1 -name '*\.[0-9]\.md' | \
-	  cut -b 5- | LC_ALL=C sort | \
-	  sed 's/\(.*\)\.\(.*\).*\.md/- "\1": "\1.\2.md"/' | \
-	  $(PYTHON) devtools/blockreplace.py mkdocs.yml manpages --language=yml --indent "          " \
-	)
-
-
-
 # Don't delete these intermediaries.
-.PRECIOUS: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) $(PYTHON_GENERATED)
+.PRECIOUS: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES)
 
 # Every single object file.
 ALL_OBJS := $(ALL_C_SOURCES:.c=.o)
@@ -466,29 +447,28 @@ else
 PYTEST_OPTS += -x
 endif
 
-# Allow for targeting specific tests by setting the PYTEST_TESTS environment variable.
-ifeq ($(PYTEST_TESTS),)
-PYTEST_TESTS = "tests/"
-endif
-
 check-units:
 
-check: check-units installcheck pytest
+check: check-units installcheck check-protos pytest
+
+check-protos: $(ALL_PROGRAMS)
+ifeq ($(PYTEST),)
+	@echo "py.test is required to run the protocol tests, please install using 'pip3 install -r requirements.txt', and rerun 'configure'."; false
+else
+ifeq ($(DEVELOPER),1)
+	@(cd external/lnprototest && PYTHONPATH=$(MY_CHECK_PYTHONPATH) LIGHTNING_SRC=../.. $(PYTEST) --runner lnprototest.clightning.Runner $(PYTEST_OPTS))
+else
+	@echo "lnprototest target requires DEVELOPER=1, skipping"
+endif
+endif
 
 pytest: $(ALL_PROGRAMS) $(DEFAULT_TARGETS) $(ALL_TEST_PROGRAMS) $(ALL_TEST_GEN)
 ifeq ($(PYTEST),)
 	@echo "py.test is required to run the integration tests, please install using 'pip3 install -r requirements.txt', and rerun 'configure'."
 	exit 1
 else
-# Explicitly hand VALGRIND so you can override on make cmd line.
-	PYTHONPATH=$(MY_CHECK_PYTHONPATH) TEST_DEBUG=1 VALGRIND=$(VALGRIND) $(PYTEST) $(PYTEST_TESTS) $(PYTEST_OPTS)
-endif
-
-check-fuzz: $(ALL_FUZZ_TARGETS)
-ifneq ($(FUZZING),0)
-	@tests/fuzz/check-fuzz.sh
-else
-	@echo "fuzzing is not enabled: first run './configure --enable-fuzzing'"
+# Explicitly hand DEVELOPER and VALGRIND so you can override on make cmd line.
+	PYTHONPATH=$(MY_CHECK_PYTHONPATH) TEST_DEBUG=1 DEVELOPER=$(DEVELOPER) VALGRIND=$(VALGRIND) $(PYTEST) tests/ $(PYTEST_OPTS)
 endif
 
 # Keep includes in alpha order.
@@ -533,6 +513,9 @@ check-whitespace/%: %
 
 check-whitespace: check-whitespace/Makefile check-whitespace/tools/check-bolt.c $(ALL_NONGEN_SRCFILES:%=check-whitespace/%)
 
+check-markdown:
+	@tools/check-markdown.sh
+
 check-spelling:
 	@tools/check-spelling.sh
 
@@ -551,7 +534,7 @@ check-python-flake8:
 	@# E731 do not assign a lambda expression, use a def
 	@# W503: line break before binary operator
 	@# E741: ambiguous variable name
-	@flake8 --ignore=E501,E731,E741,W503,F541,E275 --exclude $(shell echo ${PYTHON_GENERATED} | sed 's/ \+/,/g') ${PYSRC}
+	@flake8 --ignore=E501,E731,E741,W503,F541 --exclude $(shell echo ${PYTHON_GENERATED} | sed 's/ \+/,/g') ${PYSRC}
 
 check-pytest-pyln-proto:
 	PATH=$(PYLN_PATH) PYTHONPATH=$(MY_CHECK_PYTHONPATH) $(PYTEST) contrib/pyln-proto/tests/
@@ -559,8 +542,15 @@ check-pytest-pyln-proto:
 check-includes: check-src-includes check-hdr-includes
 	@tools/check-includes.sh
 
+# cppcheck gets confused by list_for_each(head, i, list): thinks i is uninit.
+.cppcheck-suppress:
+	@git ls-files -- "*.c" "*.h" | grep -vE '^(ccan|contrib)/' | xargs grep -n '_for_each' | sed 's/\([^:]*:.*\):.*/uninitvar:\1/' > $@
+
+check-cppcheck: .cppcheck-suppress
+	@trap 'rm -f .cppcheck-suppress' 0; git ls-files -- "*.c" "*.h" | grep -vE '^ccan/' | xargs cppcheck  ${CPPCHECK_OPTS}
+
 check-shellcheck:
-	@git ls-files -z -- "*.sh" | xargs -0 shellcheck -f gcc
+	@git ls-files -- "*.sh" | xargs shellcheck
 
 check-setup_locale:
 	@tools/check-setup_locale.sh
@@ -575,29 +565,9 @@ check-discouraged-functions:
 # since it risks overflow.
 check-amount-access:
 	@! (git grep -nE "(->|\.)(milli)?satoshis" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" | grep -v '/* Raw:')
-	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" | grep -vE "sizeof.struct amount_(m)?sat."
+	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*"
 
-repeat-doc-examples:
-	@for i in $$(seq 1 $(n)); do \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-		echo "Iteration $$i" >> tests/autogenerate-examples-repeat.log; \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-		VALGRIND=0 TIMEOUT=40 TEST_DEBUG=1 GENERATE_EXAMPLES=1 pytest -vvv tests/autogenerate-rpc-examples.py; \
-		git diff >> tests/autogenerate-examples-repeat.log; \
-		git reset --hard; \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-	done
-
-update-doc-examples:
-	TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 $(PYTEST) $(PYTEST_OPTS) --timeout=1200 tests/autogenerate-rpc-examples.py && $(MAKE) $(MSGGEN_GEN_ALL)
-
-check-doc-examples: update-doc-examples
-	git diff --exit-code HEAD
-
-# For those without working cppcheck
-check-source-no-cppcheck: check-makefile check-source-bolt check-whitespace check-spelling check-python check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access
-
-check-source: check-source-no-cppcheck
+check-source: check-makefile check-source-bolt check-whitespace check-markdown check-spelling check-python check-includes check-cppcheck check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access
 
 full-check: check check-source
 
@@ -617,15 +587,12 @@ CHECK_GEN_ALL = \
 	$(PYTHON_GENERATED) \
 	$(ALL_GEN_HEADERS) \
 	$(ALL_GEN_SOURCES) \
-	$(MSGGEN_GEN_ALL) \
 	wallet/statements_gettextgen.po \
-	doc/index.rst
-
-gen:  $(CHECK_GEN_ALL)
+	.msggen.json
 
 check-gen-updated:  $(CHECK_GEN_ALL)
 	@echo "Checking for generated files being changed by make"
-	git diff --exit-code HEAD
+	git diff --exit-code HEAD $?
 
 coverage/coverage.info: check pytest
 	mkdir coverage || true
@@ -641,10 +608,10 @@ ncc: ${TARGET_DIR}/libwally-core-build/src/libwallycore.la
 
 # Ignore test/ directories.
 TAGS:
-	$(RM) TAGS; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 etags --append
+	$(RM) TAGS; find * -name test -type d -prune -o -name '*.[ch]' -print -o -name '*.py' -print | xargs etags --append
 
 tags:
-	$(RM) tags; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 ctags --append
+	$(RM) tags; find * -name test -type d -prune -o -name '*.[ch]' -print -o -name '*.py' -print | xargs ctags --append
 
 ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS)
 
@@ -665,7 +632,7 @@ version_gen.h: $(FORCE)
 endif
 
 # That forces this rule to be run every time, too.
-header_versions_gen.h: tools/headerversions $(FORCE)
+header_versions_gen.h: tools/headerversions
 	@tools/headerversions $@
 
 # We make a static library, this way linker can discard unused parts.
@@ -683,7 +650,7 @@ $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): %: %.o
 # uses some ccan modules internally).  We want to rely on -lwallycore etc.
 # (as per EXTERNAL_LDLIBS) so we filter them out here.
 $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS):
-	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a $($(@)_LDLIBS) -o $@)
+	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a -o $@)
 
 # We special case the fuzzing target binaries, as they need to link against libfuzzer,
 # which brings its own main().
@@ -693,7 +660,7 @@ $(ALL_FUZZ_TARGETS):
 
 
 # Everything depends on the CCAN headers, and Makefile
-$(CCAN_OBJS) $(CDUMP_OBJS): $(CCAN_HEADERS) Makefile ccan_compat.h
+$(CCAN_OBJS) $(CDUMP_OBJS): $(CCAN_HEADERS) Makefile
 
 # Except for CCAN, we treat everything else as dependent on external/ bitcoin/ common/ wire/ and all generated headers, and Makefile
 $(ALL_OBJS): $(BITCOIN_HEADERS) $(COMMON_HEADERS) $(CCAN_HEADERS) $(WIRE_HEADERS) $(ALL_GEN_HEADERS) $(EXTERNAL_HEADERS) Makefile
@@ -714,7 +681,6 @@ update-ccan:
 
 # Now ALL_PROGRAMS is fully populated, we can expand it.
 all-programs: $(ALL_PROGRAMS)
-all-fuzz-programs: $(ALL_FUZZ_TARGETS)
 all-test-programs: $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS)
 default-targets: $(DEFAULT_TARGETS)
 
@@ -724,13 +690,10 @@ distclean: clean
 maintainer-clean: distclean
 	@echo 'This command is intended for maintainers to use; it'
 	@echo 'deletes files that may need special tools to rebuild.'
-	$(RM) $(PYTHON_GENERATED)
 
 # We used to have gen_ files, now we have _gen files.
-# We used to generate doc/schemas/lightning-sql.json.
 obsclean:
 	$(RM) gen_*.h */gen_*.[ch] */*/gen_*.[ch]
-	$(RM) doc/schemas/lightning-sql.json
 
 clean: obsclean
 	$(RM) libccan.a $(CCAN_OBJS) $(CDUMP_OBJS) $(ALL_OBJS)
@@ -738,7 +701,6 @@ clean: obsclean
 	$(RM) $(ALL_PROGRAMS)
 	$(RM) $(ALL_TEST_PROGRAMS)
 	$(RM) $(ALL_FUZZ_TARGETS)
-	$(RM) $(MSGGEN_GEN_ALL)
 	$(RM) ccan/tools/configurator/configurator
 	$(RM) ccan/ccan/cdump/tools/cdump-enumstr.o
 	find . -name '*gcda' -delete
@@ -748,9 +710,7 @@ clean: obsclean
 
 
 PYLNS=client proto testing
-# See doc/contribute-to-core-lightning/contributor-workflow.md
-update-versions: update-pyln-versions update-wss-proxy-version update-poetry-lock update-dot-version update-doc-examples
-
+# See doc/MAKING-RELEASES.md
 update-pyln-versions: $(PYLNS:%=update-pyln-version-%)
 
 update-pyln-version-%:
@@ -762,30 +722,35 @@ pyln-release:  $(PYLNS:%=pyln-release-%)
 pyln-release-%:
 	cd contrib/pyln-$* && $(MAKE) prod-release
 
-update-wss-proxy-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	cd plugins/wss-proxy && $(MAKE) upgrade-version
-
-update-poetry-lock:
-	poetry update wss-proxy pyln-client pyln-proto pyln-testing update-reckless-version
-
-update-reckless-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	@sed -i "s/__VERSION__ = '\([.-z]*\)'/__VERSION__ = '$(NEW_VERSION)'/" tools/reckless
-
-update-dot-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	echo $(NEW_VERSION) > .version
-
+# These must both be enabled for update-mocks
+ifeq ($(DEVELOPER)$(EXPERIMENTAL_FEATURES),11)
 update-mocks: $(ALL_TEST_PROGRAMS:%=update-mocks/%.c)
+else
+update-mocks:
+	@echo Need DEVELOPER=1 and EXPERIMENTAL_FEATURES=1 to regenerate mocks >&2; exit 1
+endif
 
 $(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) libccan.a ccan/ccan/cdump/tools/cdump-enumstr config.vars
 
-update-mocks/%: % $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES)
+update-mocks/%: %
 	@MAKE=$(MAKE) tools/update-mocks.sh "$*" $(SUPPRESS_OUTPUT)
 
-unittest/%: % bolt-precheck
-	BOLTDIR=$(LOCAL_BOLTDIR) $(VG) $(VG_TEST_ARGS) $* > /dev/null
+unittest/%: %
+	$(VG) $(VG_TEST_ARGS) $* > /dev/null
+
+# Installation directories
+exec_prefix = $(PREFIX)
+bindir = $(exec_prefix)/bin
+libexecdir = $(exec_prefix)/libexec
+pkglibexecdir = $(libexecdir)/$(PKGNAME)
+plugindir = $(pkglibexecdir)/plugins
+datadir = $(PREFIX)/share
+docdir = $(datadir)/doc/$(PKGNAME)
+mandir = $(datadir)/man
+man1dir = $(mandir)/man1
+man5dir = $(mandir)/man5
+man7dir = $(mandir)/man7
+man8dir = $(mandir)/man8
 
 # Commands
 MKDIR_P = mkdir -p
@@ -815,19 +780,17 @@ installdirs:
 
 # $(PLUGINS) is defined in plugins/Makefile.
 
-install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS)
+install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS)
 	@$(NORMAL_INSTALL)
 	$(INSTALL_PROGRAM) $(BIN_PROGRAMS) $(DESTDIR)$(bindir)
 	$(INSTALL_PROGRAM) $(PKGLIBEXEC_PROGRAMS) $(DESTDIR)$(pkglibexecdir)
-	@if [ -d "$(DESTDIR)$(plugindir)/clnrest" ]; then rm -rf $(DESTDIR)$(plugindir)/clnrest; fi
 	[ -z "$(PLUGINS)" ] || $(INSTALL_PROGRAM) $(PLUGINS) $(DESTDIR)$(plugindir)
-	for PY in $(PY_PLUGINS); do DIR=`dirname $$PY`; DST=$(DESTDIR)$(plugindir)/`basename $$DIR`; if [ -d $$DST ]; then rm -rf $$DST; fi; $(INSTALL_PROGRAM) -d $$DIR; cp -a $$DIR $$DST ; done
 
 MAN1PAGES = $(filter %.1,$(MANPAGES))
 MAN5PAGES = $(filter %.5,$(MANPAGES))
 MAN7PAGES = $(filter %.7,$(MANPAGES))
 MAN8PAGES = $(filter %.8,$(MANPAGES))
-DOC_DATA = README.md LICENSE
+DOC_DATA = README.md doc/INSTALL.md doc/HACKING.md LICENSE
 
 install-data: installdirs $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA)
 	@$(NORMAL_INSTALL)
@@ -839,27 +802,6 @@ install-data: installdirs $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(
 
 install: install-program install-data
 
-# Non-artifacts that are needed for testing. These are added to the
-# testpack.tar, used to transfer things between builder and tester
-# phase. If you get a missing file/executable while testing on CI it
-# is likely missing from this variable.
-TESTBINS = \
-	$(CLN_PLUGIN_EXAMPLES) \
-	tests/plugins/test_libplugin \
-	tests/plugins/channeld_fakenet \
-	tests/plugins/test_selfdisable_after_getmanifest \
-	tools/hsmtool
-
-# The testpack is used in CI to transfer built artefacts between the
-# build and the test phase. This is necessary because the fixtures in
-# `tests/` explicitly use the binaries built in the current directory
-# rather than using `$PATH`, as that may pick up some other installed
-# version of `lightningd` leading to bogus results. We bundle up all
-# built artefacts here, and will unpack them on the tester (overlaying
-# on top of the checked out repo as if we had just built it in place).
-testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS)
-	tar -caf $@ $^
-
 uninstall:
 	@$(NORMAL_UNINSTALL)
 	@for f in $(BIN_PROGRAMS); do \
@@ -870,10 +812,6 @@ uninstall:
 	  $(ECHO) rm -f $(DESTDIR)$(plugindir)/`basename $$f`; \
 	  rm -f $(DESTDIR)$(plugindir)/`basename $$f`; \
 	done
-	@for f in $(PY_PLUGINS); do \
-	  $(ECHO) rm -rf $(DESTDIR)$(plugindir)/$$(basename $$(dirname $$f)); \
-	  rm -rf $(DESTDIR)$(plugindir)/$$(basename $$(dirname $$f)); \
-	done
 	@for f in $(PKGLIBEXEC_PROGRAMS); do \
 	  $(ECHO) rm -f $(DESTDIR)$(pkglibexecdir)/`basename $$f`; \
 	  rm -f $(DESTDIR)$(pkglibexecdir)/`basename $$f`; \
@@ -910,11 +848,8 @@ installcheck: all-programs
 	fi
 	@rm -rf testinstall || true
 
-version:
-	@echo ${VERSION}
-
 .PHONY: installdirs install-program install-data install uninstall \
-	installcheck ncc bin-tarball show-flags version
+	installcheck ncc bin-tarball show-flags
 
 # Make a tarball of opt/clightning/, optionally with label for distribution.
 ifneq ($(VERSION),)

README.md

@@ -15,38 +15,44 @@ Core Lightning (previously c-lightning) is a lightweight, highly customizable an
     * [Pruning](#pruning)
     * [HD wallet encryption](#hd-wallet-encryption)
 	* [Developers](#developers)
-* [Documentation](https://docs.corelightning.org/docs)
+* [Documentation](https://lightning.readthedocs.io/)
 
 ## Project Status
 
-[![Continuous Integration][actions-badge]][actions]
-[![Pull Requests Welcome][prs-badge]][prs]
-[![Documentation Status][docs-badge]][docs]
-[![BoL2][bol2-badge]][bol2]
-[![Telegram][telegram-badge]][telegram]
-[![Discord][discord-badge]][discord]
-[![Irc][IRC-badge]][IRC]
+[![Continuous Integration](https://github.com/ElementsProject/lightning/workflows/Continuous%20Integration/badge.svg)][actions]
+[![Pull Requests Welcome][prs]][prs-link]
+[![Irc][IRC]][IRC-link]
+[![Documentation Status](https://readthedocs.org/projects/lightning/badge/?version=docs)][docs]
 
 This implementation has been in production use on the Bitcoin mainnet since early 2018, with the launch of the [Blockstream Store][blockstream-store-blog].
-We recommend getting started by experimenting on `testnet` (`testnet4` or `regtest`), but the implementation is considered stable and can be safely used on mainnet.
-
-## Reach Out to Us
+We recommend getting started by experimenting on `testnet` (or `regtest`), but the implementation is considered stable and can be safely used on mainnet.
 
 Any help testing the implementation, reporting bugs, or helping with outstanding issues is very welcome.
-Don't hesitate to reach out to us on [Build-on-L2][bol2], or on the implementation-specific [mailing list][ml1], or on [CLN Discord][discord], or on [CLN Telegram][telegram], or on IRC at [dev][irc1]/[gen][irc2] channel.
+Don't hesitate to reach out to us on IRC at [#lightning-dev @ libera.chat][irc1], [#c-lightning @ libera.chat][irc2], or on the implementation-specific mailing list [c-lightning@lists.ozlabs.org][ml1], or on the Lightning Network-wide mailing list [lightning-dev@lists.linuxfoundation.org][ml2], or on Discord [core-lightning][discord], or on Telegram [Core Lightning][telegram].
 
 ## Getting Started
 
-Core Lightning only works on Linux and macOS, and requires a locally (or remotely) running `bitcoind` (version 25.0 or above) that is fully caught up with the network you're running on, and relays transactions (ie with `blocksonly=0`).
+Core Lightning only works on Linux and macOS, and requires a locally (or remotely) running `bitcoind` (version 0.16 or above) that is fully caught up with the network you're running on, and relays transactions (ie with `blocksonly=0`).
 Pruning (`prune=n` option in `bitcoin.conf`) is partially supported, see [here](#pruning) for more details.
 
 ### Installation
 
-There are 3 supported installation options:
+There are 4 supported installation options:
 
+ - Installation from the [Ubuntu PPA][ppa].
  - Installation of a pre-compiled binary from the [release page][releases] on GitHub.
  - Using one of the [provided docker images][dockerhub] on the Docker Hub.
- - Compiling the source code yourself as described in the [installation documentation](doc/getting-started/getting-started/installation.md).
+ - Compiling the source code yourself as described in the [installation documentation](doc/INSTALL.md).
+
+For the impatient here's the gist of it for Ubuntu:
+
+```bash
+sudo apt-get install -y software-properties-common
+sudo add-apt-repository -u ppa:lightningnetwork/ppa
+sudo apt-get install lightningd snapd
+sudo snap install bitcoin-core
+sudo ln -s /snap/bitcoin-core/current/bin/bitcoin{d,-cli} /usr/local/bin/
+```
 
 ### Starting `lightningd`
 
@@ -60,6 +66,13 @@ of the `startup_regtest.sh` file for details on how to use it.
 . contrib/startup_regtest.sh
 ```
 
+Note that your local nodeset will be much faster/more responsive if
+you've configured your node to expose the developer options, e.g.
+
+```bash
+./configure --enable-developer
+```
+
 #### Mainnet Option
 To test with real bitcoin,  you will need to have a local `bitcoind` node running:
 
@@ -78,7 +91,7 @@ You can start `lightningd` with the following command:
 lightningd --network=bitcoin --log-level=debug
 ```
 
-This creates a `.lightning/` subdirectory in your home directory: see `man -l doc/lightningd.8` (or https://docs.corelightning.org/docs) for more runtime options.
+This creates a `.lightning/` subdirectory in your home directory: see `man -l doc/lightningd.8` (or https://lightning.readthedocs.io/) for more runtime options.
 
 ### Using The JSON-RPC Interface
 
@@ -104,12 +117,17 @@ Once you've started for the first time, there's a script called
 the lightning network.
 
 There are also numerous plugins available for Core Lightning which add
-capabilities: in particular there's a collection at: https://github.com/lightningd/plugins
+capabilities: in particular there's a collection at:
+
+	https://github.com/lightningd/plugins
+
+Including [helpme][helpme-github] which guides you through setting up
+your first channels and customizing your node.
 
 For a less reckless experience, you can encrypt the HD wallet seed:
  see [HD wallet encryption](#hd-wallet-encryption).
 
-You can also chat to other users at Discord [core-lightning][discord];
+You can also chat to other users at [#c-lightning @ libera.chat][irc2];
 we are always happy to help you get started!
 
 
@@ -125,11 +143,11 @@ lightning-cli newaddr
 
 `lightningd` will register the funds once the transaction is confirmed.
 
-Alternatively you can generate a taproot address should your source of funds support it:
+You may need to generate a p2sh-segwit address if the faucet does not support bech32:
 
 ```bash
-# Return a taproot address
-lightning-cli newaddr p2tr
+# Return a p2sh-segwit address
+lightning-cli newaddr p2sh-segwit
 ```
 
 Confirm `lightningd` got funds by:
@@ -184,8 +202,6 @@ Command line options will always override the values in the configuration file.
 To use a configuration file, create a file named `config` within your top-level lightning directory or network subdirectory
 (eg. `~/.lightning/config` or `~/.lightning/bitcoin/config`).  See `man -l doc/lightningd-config.5`.
 
-A sample configuration file is available at `contrib/config-example`.
-
 ## Further information
 
 ### Pruning
@@ -205,28 +221,25 @@ If you encrypt your `hsm_secret`, you will have to pass the `--encrypted-hsm` st
 
 ### Developers
 
-Developers wishing to contribute should start with the developer guide [here](doc/contribute-to-core-lightning/coding-style-guidelines.md).
+Developers wishing to contribute should start with the developer guide [here](doc/HACKING.md).
+You should also configure with `--enable-developer` to get additional checks and options.
 
 [blockstream-store-blog]: https://blockstream.com/2018/01/16/en-lightning-charge/
 [std]: https://github.com/lightning/bolts
-[prs-badge]: https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat
-[prs]: http://makeapullrequest.com
-[bol2-badge]: https://badgen.net/badge/BoL2/chat/blue
-[bol2]: https://community.corelightning.org
-[ml1]: https://lists.ozlabs.org/listinfo/c-lightning
-[discord-badge]: https://badgen.net/badge/Discord/chat/blue
-[discord]: https://discord.gg/mE9s4rc5un
-[telegram-badge]: https://badgen.net/badge/Telegram/chat/blue
-[telegram]: https://t.me/lightningd
-[IRC-badge]: https://img.shields.io/badge/IRC-chat-blue.svg
-[IRC]: https://web.libera.chat/#c-lightning
+[prs]: https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat
+[prs-link]: http://makeapullrequest.com
+[IRC]: https://img.shields.io/badge/chat-on%20libera-brightgreen.svg
+[IRC-link]: https://web.libera.chat/#c-lightning
 [irc1]: https://web.libera.chat/#lightning-dev
 [irc2]: https://web.libera.chat/#c-lightning
-[docs-badge]: https://readthedocs.org/projects/lightning/badge/?version=docs
-[docs]: https://docs.corelightning.org/docs
+[ml1]: https://lists.ozlabs.org/listinfo/c-lightning
+[ml2]: https://lists.linuxfoundation.org/mailman/listinfo/lightning-dev
+[discord]: https://discord.gg/mE9s4rc5un
+[telegram]: https://t.me/lightningd
+[docs]: https://lightning.readthedocs.org
+[ppa]: https://launchpad.net/~lightningnetwork/+archive/ubuntu/ppa
 [releases]: https://github.com/ElementsProject/lightning/releases
 [dockerhub]: https://hub.docker.com/r/elementsproject/lightningd/
 [jsonrpcspec]: https://www.jsonrpc.org/specification
 [helpme-github]: https://github.com/lightningd/plugins/tree/master/helpme
-[actions-badge]: https://github.com/ElementsProject/lightning/workflows/Continuous%20Integration/badge.svg
 [actions]: https://github.com/ElementsProject/lightning/actions

SECURITY.md

@@ -1,35 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-We have a 3 month release cycle, and the last two versions are supported.
-
-## Reporting a Vulnerability
-
-To report security vulnerabilities, please send an email to one of the following addresses:
-- `rusty@rustcorp.com.au`
-- `security@blockstream.com`
-
-Note: These email addresses are exclusively for vulnerability reporting.
-
-For all other inquiries/communication, please refer to the [Reach Out to Us](https://github.com/ElementsProject/lightning?tab=readme-ov-file#reach-out-to-us) section in our README.
-
-## Signatures For Releases
-
-The following keys may be used to communicate sensitive information to
-developers, and to validate signatures on releases:
-
-| Name | Email | Fingerprint |
-|------|-------|-------------|
-| Blockstream Security Reporting | `security@blockstream.com` | 1176 542D A98E 71E1 3372  2EF7 4AC8 CC88 6844 A2D6 |
-| Rusty Russell | `rusty@rustcorp.com.au` | 15EE 8D6C AB0E 7F0C F999  BFCB D920 0E6C D1AD B8F1 |
-| Christian Decker | `decker@blockstream.com` | B731 AAC5 21B0 1385 9313  F674 A26D 6D9F E088 ED58 |
-| Lisa Neigut | `niftynei@gmail.com` | 30DE 693A E0DE 9E37 B3E7  EB6B BFF0 F678 10C1 EED1 |
-| Alex Myers | `alex@endothermic.dev` | 0437 4E42 789B BBA9 462E  4767 F3BF 63F2 7474 36AB |
-| Peter Neuroth | `pet.v.ne@gmail.com` | 653B 19F3 3DF7 EFF3 E9D1  C94C C3F2 1EE3 87FF 4CD2 |
-| Shahana Farooqui | `sfarooqui@blockstream.com` | FE13 58EB 7793 51DB 24E5  555A A327 573C 9758 9BF5 |
-| Blockstream CLN Release | `cln@blockstream.com` | 616C 52F9 9D06 12B2 A151  B107 4129 A994 AA7E 9852 |
-
-You can import a key by running the following command with that individual’s fingerprint: 
-`gpg --keyserver hkps://keys.openpgp.org --recv-keys "<fingerprint>"`. 
-Ensure that you put quotes around fingerprints containing spaces.

action.yml

@@ -0,0 +1,6 @@
+---
+name: 'Lightning CI'
+description: 'A preconfigured container with all Core Lightning dependencies'
+runs:
+  using: 'docker'
+  image: 'contrib/Dockerfile.tester'

bitcoin/base58.c

@@ -47,14 +47,16 @@ static bool from_base58(u8 *version,
 			struct ripemd160 *rmd,
 			const char *base58, size_t base58_len)
 {
-	/* Initialize to avoid memcheck complaining if decoding a short value */
-	u8 buf[1 + sizeof(*rmd) + 4] = { 0 };
-	const size_t buflen = sizeof(buf);
-	const uint32_t flags = BASE58_FLAG_CHECKSUM;
+	u8 buf[1 + sizeof(*rmd) + 4];
+	/* Avoid memcheck complaining if decoding resulted in a short value */
+	size_t buflen = sizeof(buf);
+	memset(buf, 0, buflen);
+	char *terminated_base58 = tal_dup_arr(NULL, char, base58, base58_len, 1);
+	terminated_base58[base58_len] = '\0';
 
 	size_t written = 0;
-	int r = wally_base58_n_to_bytes(base58, base58_len, flags,
-					buf, buflen, &written);
+	int r = wally_base58_to_bytes(terminated_base58, BASE58_FLAG_CHECKSUM, buf, buflen, &written);
+	tal_free(terminated_base58);
 	if (r != WALLY_OK || written > buflen) {
 		return false;
 	}

bitcoin/block.c

@@ -5,7 +5,7 @@
 #include <bitcoin/tx.h>
 #include <ccan/mem/mem.h>
 #include <ccan/str/hex/hex.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 
 /* Sets *cursor to NULL and returns NULL when a pull fails. */
 static const u8 *pull(const u8 **cursor, size_t *max, void *copy, size_t n)
@@ -210,7 +210,7 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams,
 	b->tx = tal_arr(b, struct bitcoin_tx *, num);
 	b->txids = tal_arr(b, struct bitcoin_txid, num);
 	for (i = 0; i < num; i++) {
-		b->tx[i] = pull_bitcoin_tx_only(b->tx, &p, &len);
+		b->tx[i] = pull_bitcoin_tx(b->tx, &p, &len);
 		b->tx[i]->chainparams = chainparams;
 		bitcoin_txid(b->tx[i], &b->txids[i]);
 	}
@@ -237,14 +237,15 @@ static bool bitcoin_blkid_to_hex(const struct bitcoin_blkid *blockid,
 	return bitcoin_txid_to_hex(&fake_txid, hexstr, hexstr_len);
 }
 
-char *fmt_bitcoin_blkid(const tal_t *ctx,
-			const struct bitcoin_blkid *blkid)
+static char *fmt_bitcoin_blkid(const tal_t *ctx,
+			       const struct bitcoin_blkid *blkid)
 {
 	char *hexstr = tal_arr(ctx, char, hex_str_size(sizeof(*blkid)));
 
 	bitcoin_blkid_to_hex(blkid, hexstr, hex_str_size(sizeof(*blkid)));
 	return hexstr;
 }
+REGISTER_TYPE_TO_STRING(bitcoin_blkid, fmt_bitcoin_blkid);
 
 void fromwire_bitcoin_blkid(const u8 **cursor, size_t *max,
 			    struct bitcoin_blkid *blkid)

bitcoin/block.h

@@ -53,7 +53,4 @@ void fromwire_chainparams(const u8 **cursor, size_t *max,
 			  const struct chainparams **chainparams);
 void towire_chainparams(u8 **cursor, const struct chainparams *chainparams);
 
-char *fmt_bitcoin_blkid(const tal_t *ctx,
-			const struct bitcoin_blkid *blkid);
-
 #endif /* LIGHTNING_BITCOIN_BLOCK_H */

bitcoin/chainparams.c

@@ -50,7 +50,6 @@ const struct chainparams networks[] = {
       */
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      /* "Lightning Charge Powers Developers & Blockstream Store" */
      .when_lightning_became_cool = 504500,
      .p2pkh_version = 0,
@@ -77,7 +76,6 @@ const struct chainparams networks[] = {
      .dust_limit = { 546 },
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 111,
      .p2sh_version = 196,
@@ -104,7 +102,6 @@ const struct chainparams networks[] = {
      .dust_limit = { 546 },
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 111,
      .p2sh_version = 196,
@@ -129,33 +126,6 @@ const struct chainparams networks[] = {
      .dust_limit = { 546 },
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
-     .p2pkh_version = 111,
-     .p2sh_version = 196,
-     .testnet = true,
-     .fee_asset_tag = NULL,
-     .bip32_key_version = {.bip32_pubkey_version = BIP32_VER_TEST_PUBLIC,
-			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
-     .is_elements = false},
-    {.network_name = "testnet4",
-     .onchain_hrp = "tb",
-     .lightning_hrp = "tb",
-     .bip70_name = "testnet4",
-     // 00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043
-     .genesis_blockhash = {{{.u.u8 = {0x43, 0xf0, 0x8b, 0xda, 0xb0, 0x50, 0xe3,
-				      0x5b, 0x56, 0x7c, 0x86, 0x4b, 0x91, 0xf4,
-				      0x7f, 0x50, 0xae, 0x72, 0x5a, 0xe2, 0xde,
-				      0x53, 0xbc, 0xfb, 0xba, 0xf2, 0x84, 0xda,
-				      0x00, 0x00, 0x00, 0x00}}}},
-     .rpc_port = 48332,
-     .ln_port = 49735,
-     .cli = "bitcoin-cli",
-     .cli_args = "-testnet4",
-     .cli_min_supported_version = 150000,
-     .dust_limit = { 546 },
-     .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
-     .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .p2pkh_version = 111,
      .p2sh_version = 196,
      .testnet = true,
@@ -180,7 +150,6 @@ const struct chainparams networks[] = {
      .dust_limit = { 100000 },
      .max_funding = AMOUNT_SAT_INIT(60 * ((1 << 24) - 1)),
      .max_payment = AMOUNT_MSAT_INIT(60 * 0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1320000,
      .p2pkh_version = 48,
      .p2sh_version = 50,
@@ -206,7 +175,6 @@ const struct chainparams networks[] = {
      .dust_limit = { 100000 },
      .max_funding = AMOUNT_SAT_INIT(60 * ((1 << 24) - 1)),
      .max_payment = AMOUNT_MSAT_INIT(60 * 0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 111,
      .p2sh_version = 58,
@@ -231,7 +199,6 @@ const struct chainparams networks[] = {
      .dust_limit = {546},
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 91,
      .p2sh_version = 75,
@@ -256,7 +223,6 @@ const struct chainparams networks[] = {
      .dust_limit = {546},
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 57,
      .p2sh_version = 39,

bitcoin/chainparams.h

@@ -29,7 +29,7 @@ struct chainparams {
 	 *
 	 * - Bitcoin mainet with port number 9735 or the corresponding hexadecimal `0x2607`;
 	 * - Bitcoin testnet with port number 19735 (`0x4D17`);
-	 * - Bitcoin signet with port number 39735 (`0x9B37`).
+	 * - Bitcoin signet with port number 39735 (`0xF87`).
 	 */
 	const int ln_port;
 	const char *cli;
@@ -39,8 +39,6 @@ struct chainparams {
 	const struct amount_sat dust_limit;
 	const struct amount_sat max_funding;
 	const struct amount_msat max_payment;
-	/* Total coins in network */
-	const struct amount_sat max_supply;
 	const u32 when_lightning_became_cool;
 	const u8 p2pkh_version;
 	const u8 p2sh_version;

bitcoin/feerate.c

@@ -1,12 +1,8 @@
 #include "config.h"
-#include <assert.h>
 #include <bitcoin/feerate.h>
 
 u32 feerate_from_style(u32 feerate, enum feerate_style style)
 {
-	/* Make sure it's called somewhere! */
-	assert(feerate_floor_check() == FEERATE_FLOOR);
-
 	switch (style) {
 	case FEERATE_PER_KSIPA:
 		return feerate;

bitcoin/feerate.h

@@ -39,7 +39,7 @@ enum feerate_style {
 	FEERATE_PER_KBYTE
 };
 
-static inline u32 feerate_floor_check(void)
+static inline u32 feerate_floor(void)
 {
 	/* Assert that bitcoind will see this as above minRelayTxFee */
 	BUILD_ASSERT(FEERATE_BITCOIND_SEES(FEERATE_FLOOR, MINIMUM_TX_WEIGHT)

bitcoin/locktime.c

@@ -2,6 +2,7 @@
 #include <assert.h>
 #include <bitcoin/locktime.h>
 #include <ccan/tal/str/str.h>
+#include <common/type_to_string.h>
 
 #define SECONDS_POINT 500000000
 

bitcoin/preimage.c

@@ -1,6 +1,5 @@
 #include "config.h"
 #include <bitcoin/preimage.h>
-#include <common/utils.h>
 #include <wire/wire.h>
 
 void fromwire_preimage(const u8 **cursor, size_t *max, struct preimage *preimage)
@@ -13,7 +12,4 @@ void towire_preimage(u8 **pptr, const struct preimage *preimage)
 	towire(pptr, preimage, sizeof(*preimage));
 }
 
-char *fmt_preimage(const tal_t *ctx, const struct preimage *preimage)
-{
-	return tal_hexstr(ctx, preimage, sizeof(*preimage));
-}
+

bitcoin/preimage.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/short_types/short_types.h>
 #include <ccan/structeq/structeq.h>
-#include <ccan/tal/tal.h>
 
 struct preimage {
 	u8 r[32];
@@ -14,5 +13,4 @@ STRUCTEQ_DEF(preimage, 0, r);
 void fromwire_preimage(const u8 **cursor, size_t *max, struct preimage *preimage);
 void towire_preimage(u8 **pptr, const struct preimage *preimage);
 
-char *fmt_preimage(const tal_t *ctx, const struct preimage *preimage);
 #endif /* LIGHTNING_BITCOIN_PREIMAGE_H */

bitcoin/privkey.c

@@ -1,10 +1,10 @@
 #include "config.h"
 #include <bitcoin/privkey.h>
 #include <ccan/str/hex/hex.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
-char *fmt_privkey(const tal_t *ctx, const struct privkey *secret)
+static char *privkey_to_hexstr(const tal_t *ctx, const struct privkey *secret)
 {
 	/* Bitcoin appends "01" to indicate the pubkey is compressed. */
 	char *str = tal_arr(ctx, char, hex_str_size(sizeof(*secret) + 1));
@@ -12,11 +12,8 @@ char *fmt_privkey(const tal_t *ctx, const struct privkey *secret)
 	strcat(str, "01");
 	return str;
 }
-
-char *fmt_secret(const tal_t *ctx, const struct secret *secret)
-{
-	return tal_hexstr(ctx, secret, sizeof(*secret));
-}
+REGISTER_TYPE_TO_STRING(privkey, privkey_to_hexstr);
+REGISTER_TYPE_TO_HEXSTR(secret);
 
 bool secret_eq_consttime(const struct secret *a, const struct secret *b)
 {

bitcoin/privkey.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/short_types/short_types.h>
 #include <ccan/structeq/structeq.h>
-#include <ccan/tal/tal.h>
 
 #define PRIVKEY_LEN 32
 
@@ -26,7 +25,4 @@ void fromwire_privkey(const u8 **cursor, size_t *max, struct privkey *privkey);
 void towire_privkey(u8 **pptr, const struct privkey *privkey);
 void towire_secret(u8 **pptr, const struct secret *secret);
 
-char *fmt_privkey(const tal_t *ctx, const struct privkey *privkey);
-char *fmt_secret(const tal_t *ctx, const struct secret *secret);
-
 #endif /* LIGHTNING_BITCOIN_PRIVKEY_H */

bitcoin/psbt.c

@@ -4,11 +4,10 @@
 #include <bitcoin/psbt.h>
 #include <bitcoin/pubkey.h>
 #include <bitcoin/script.h>
-#include <bitcoin/varint.h>
 #include <ccan/ccan/array_size/array_size.h>
 #include <ccan/ccan/mem/mem.h>
 #include <ccan/tal/str/str.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wally_psbt.h>
 #include <wire/wire.h>
 
@@ -18,25 +17,47 @@ static void psbt_destroy(struct wally_psbt *psbt)
 	wally_psbt_free(psbt);
 }
 
-struct wally_psbt *create_psbt(const tal_t *ctx, size_t num_inputs, size_t num_outputs, u32 locktime)
+static struct wally_psbt *init_psbt(const tal_t *ctx, size_t num_inputs, size_t num_outputs)
 {
-	const u32 init_flags = is_elements(chainparams) ? WALLY_PSBT_INIT_PSET : 0;
-	struct wally_psbt *psbt;
 	int wally_err;
+	struct wally_psbt *psbt;
 
 	tal_wally_start();
-	wally_err = wally_psbt_init_alloc(2, num_inputs, num_outputs, 0, init_flags, &psbt);
+	if (is_elements(chainparams))
+		wally_err = wally_psbt_elements_init_alloc(0, num_inputs, num_outputs, 0, &psbt);
+	else
+		wally_err = wally_psbt_init_alloc(0, num_inputs, num_outputs, 0, &psbt);
 	assert(wally_err == WALLY_OK);
-	wally_psbt_set_fallback_locktime(psbt, locktime);
-	/* By default we are modifying them internally; allow it */
-	wally_psbt_set_tx_modifiable_flags(psbt, WALLY_PSBT_TXMOD_INPUTS | WALLY_PSBT_TXMOD_OUTPUTS);
 	tal_add_destructor(psbt, psbt_destroy);
 	tal_wally_end_onto(ctx, psbt, struct wally_psbt);
 
 	return psbt;
 }
 
-struct wally_psbt *clone_psbt(const tal_t *ctx, const struct wally_psbt *psbt)
+struct wally_psbt *create_psbt(const tal_t *ctx, size_t num_inputs, size_t num_outputs, u32 locktime)
+{
+	int wally_err;
+	struct wally_tx *wtx;
+	struct wally_psbt *psbt;
+
+	tal_wally_start();
+	if (wally_tx_init_alloc(WALLY_TX_VERSION_2, locktime, num_inputs, num_outputs, &wtx) != WALLY_OK)
+		abort();
+	/* wtx is freed below */
+	tal_wally_end(NULL);
+
+	psbt = init_psbt(ctx, num_inputs, num_outputs);
+
+	tal_wally_start();
+	wally_err = wally_psbt_set_global_tx(psbt, wtx);
+	assert(wally_err == WALLY_OK);
+	tal_wally_end(psbt);
+
+	wally_tx_free(wtx);
+	return psbt;
+}
+
+struct wally_psbt *clone_psbt(const tal_t *ctx, struct wally_psbt *psbt)
 {
 	struct wally_psbt *clone;
 	tal_wally_start();
@@ -51,17 +72,17 @@ struct wally_psbt *new_psbt(const tal_t *ctx, const struct wally_tx *wtx)
 	struct wally_psbt *psbt;
 	int wally_err;
 
-	psbt = create_psbt(ctx, wtx->num_inputs, wtx->num_outputs, wtx->locktime);
+	psbt = init_psbt(ctx, wtx->num_inputs, wtx->num_outputs);
 
 	tal_wally_start();
-
-	/* locktime and modifiable flags are set in create_psbt */
-	wally_psbt_set_tx_version(psbt, wtx->version);
+	/* Set directly: avoids psbt checks for non-NULL scripts/witnesses */
+	wally_err = wally_tx_clone_alloc(wtx, 0, &psbt->tx);
+	assert(wally_err == WALLY_OK);
+	/* Inputs/outs are pre-allocated above, 'add' them as empty dummies */
+	psbt->num_inputs = wtx->num_inputs;
+	psbt->num_outputs = wtx->num_outputs;
 
 	for (size_t i = 0; i < wtx->num_inputs; i++) {
-		wally_err = wally_psbt_add_tx_input_at(psbt, i, 0, &wtx->inputs[i]);
-		assert(wally_err == WALLY_OK);
-
 		/* add these scripts + witnesses to the psbt */
 		if (wtx->inputs[i].script) {
 			wally_err =
@@ -78,103 +99,10 @@ struct wally_psbt *new_psbt(const tal_t *ctx, const struct wally_tx *wtx)
 		}
 	}
 
-	for (size_t i = 0; i < wtx->num_outputs; i++) {
-		wally_psbt_add_tx_output_at(psbt, i, 0, &wtx->outputs[i]);
-	}
-
 	tal_wally_end(psbt);
 	return psbt;
 }
 
-struct wally_psbt *combine_psbt(const tal_t *ctx,
-				const struct wally_psbt *psbt0,
-				const struct wally_psbt *psbt1)
-{
-	struct wally_psbt *combined_psbt;
-	tal_wally_start();
-	if (wally_psbt_clone_alloc(psbt0, 0, &combined_psbt) != WALLY_OK)
-		abort();
-	if (wally_psbt_combine(combined_psbt, psbt1) != WALLY_OK) {
-		tal_wally_end_onto(ctx, combined_psbt, struct wally_psbt);
-		return tal_free(combined_psbt);
-	}
-	tal_wally_end_onto(ctx, combined_psbt, struct wally_psbt);
-	return combined_psbt;
-}
-
-static bool parent_or_grandparent(const tal_t *goal, const tal_t *child)
-{
-	const tal_t *parent = tal_parent(child);
-	if (!parent)
-		return false;
-	return parent == goal || parent_or_grandparent(goal, parent);
-}
-
-#define NULL_OR_MATCH_P(item, parent) \
-	((item) == NULL || parent_or_grandparent((parent), (item)))
-
-static void audit_map(const tal_t *ctx, const struct wally_map *map)
-{
-	assert(NULL_OR_MATCH_P(map->items, ctx));
-	for (size_t i = 0; i < map->num_items; i++) {
-		assert(NULL_OR_MATCH_P(map->items[i].key, ctx));
-		assert(NULL_OR_MATCH_P(map->items[i].value, ctx));
-		assert(!map->items[i].key
-			|| tal_bytelen(map->items[i].key)
-				== map->items[i].key_len);
-		assert(!map->items[i].value
-			|| tal_bytelen(map->items[i].value)
-				== map->items[i].value_len);
-	}
-}
-
-void audit_psbt(const tal_t *ctx, const struct wally_psbt *psbt)
-{
-	assert(psbt);
-	assert(NULL_OR_MATCH_P(psbt->tx, ctx));
-	assert(NULL_OR_MATCH_P(psbt->inputs, ctx));
-	assert(NULL_OR_MATCH_P(psbt->outputs, ctx));
-	audit_map(ctx, &psbt->unknowns);
-	audit_map(ctx, &psbt->global_xpubs);
-#ifndef WALLY_ABI_NO_ELEMENTS
-	audit_map(ctx, &psbt->global_scalars);
-#endif
-	for (size_t i = 0; i < psbt->num_inputs; i++) {
-		assert(NULL_OR_MATCH_P(psbt->inputs[i].utxo, ctx));
-		assert(NULL_OR_MATCH_P(psbt->inputs[i].witness_utxo, ctx));
-		assert(NULL_OR_MATCH_P(psbt->inputs[i].final_witness, ctx));
-		audit_map(ctx, &psbt->inputs[i].keypaths);
-		audit_map(ctx, &psbt->inputs[i].signatures);
-		audit_map(ctx, &psbt->inputs[i].unknowns);
-		audit_map(ctx, &psbt->inputs[i].preimages);
-		audit_map(ctx, &psbt->inputs[i].psbt_fields);
-		audit_map(ctx, &psbt->inputs[i].taproot_leaf_signatures);
-		audit_map(ctx, &psbt->inputs[i].taproot_leaf_scripts);
-		audit_map(ctx, &psbt->inputs[i].taproot_leaf_hashes);
-		audit_map(ctx, &psbt->inputs[i].taproot_leaf_paths);
-		/* DTODO: Investigate if taproot wally maps have child maps */
-#ifndef WALLY_ABI_NO_ELEMENTS
-		assert(NULL_OR_MATCH_P(psbt->inputs[i].pegin_tx, ctx));
-		assert(NULL_OR_MATCH_P(psbt->inputs[i].pegin_witness, ctx));
-		audit_map(ctx, &psbt->inputs[i].pset_fields);
-#endif
-	}
-	for (size_t i = 0; i < psbt->num_outputs; i++) {
-		assert(NULL_OR_MATCH_P(psbt->outputs[i].script, ctx));
-		assert(psbt->outputs[i].script_len
-			== tal_bytelen(psbt->outputs[i].script));
-		audit_map(ctx, &psbt->outputs[i].keypaths);
-		audit_map(ctx, &psbt->outputs[i].unknowns);
-		audit_map(ctx, &psbt->outputs[i].psbt_fields);
-		audit_map(ctx, &psbt->outputs[i].taproot_tree);
-		audit_map(ctx, &psbt->outputs[i].taproot_leaf_hashes);
-		audit_map(ctx, &psbt->outputs[i].taproot_leaf_paths);
-#ifndef WALLY_ABI_NO_ELEMENTS
-		audit_map(ctx, &psbt->outputs[i].pset_fields);
-#endif
-	}
-}
-
 bool psbt_is_finalized(const struct wally_psbt *psbt)
 {
 	size_t is_finalized;
@@ -184,14 +112,14 @@ bool psbt_is_finalized(const struct wally_psbt *psbt)
 }
 
 struct wally_psbt_input *psbt_add_input(struct wally_psbt *psbt,
-					const struct wally_tx_input *input,
+					struct wally_tx_input *input,
 				       	size_t insert_at)
 {
 	const u32 flags = WALLY_PSBT_FLAG_NON_FINAL; /* Skip script/witness */
 	int wally_err;
 
 	tal_wally_start();
-	wally_err = wally_psbt_add_tx_input_at(psbt, insert_at, flags, input);
+	wally_err = wally_psbt_add_input_at(psbt, insert_at, flags, input);
 	assert(wally_err == WALLY_OK);
 	tal_wally_end(psbt);
 	return &psbt->inputs[insert_at];
@@ -231,7 +159,7 @@ struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt,
 			abort();
 	}
 
-	wally_err = wally_psbt_add_tx_input_at(psbt, input_num, flags, tx_in);
+	wally_err = wally_psbt_add_input_at(psbt, input_num, flags, tx_in);
 	assert(wally_err == WALLY_OK);
 	wally_tx_input_free(tx_in);
 	tal_wally_end(psbt);
@@ -267,7 +195,7 @@ struct wally_psbt_output *psbt_add_output(struct wally_psbt *psbt,
 	int wally_err;
 
 	tal_wally_start();
-	wally_err = wally_psbt_add_tx_output_at(psbt, insert_at, 0, output);
+	wally_err = wally_psbt_add_output_at(psbt, insert_at, 0, output);
 	assert(wally_err == WALLY_OK);
 	tal_wally_end(psbt);
 	return &psbt->outputs[insert_at];
@@ -277,9 +205,13 @@ struct wally_psbt_output *psbt_append_output(struct wally_psbt *psbt,
 					     const u8 *script,
 					     struct amount_sat amount)
 {
-	return psbt_insert_output(psbt, script, amount, psbt->num_outputs);
-}
+	struct wally_psbt_output *out;
+	struct wally_tx_output *tx_out = wally_tx_output(NULL, script, amount);
 
+	out = psbt_add_output(psbt, tx_out, psbt->tx->num_outputs);
+	wally_tx_output_free(tx_out);
+	return out;
+}
 struct wally_psbt_output *psbt_insert_output(struct wally_psbt *psbt,
 					     const u8 *script,
 					     struct amount_sat amount,
@@ -301,7 +233,7 @@ void psbt_rm_output(struct wally_psbt *psbt,
 }
 
 void psbt_input_add_pubkey(struct wally_psbt *psbt, size_t in,
-			   const struct pubkey *pubkey, bool is_taproot)
+			   const struct pubkey *pubkey)
 {
 	int wally_err;
 	u32 empty_path[1] = {0};
@@ -323,20 +255,11 @@ void psbt_input_add_pubkey(struct wally_psbt *psbt, size_t in,
 	pubkey_to_der(pk_der, pubkey);
 
 	tal_wally_start();
-	if (is_taproot) {
-		wally_err = wally_psbt_input_taproot_keypath_add(&psbt->inputs[in],
-								  pk_der + 1, 32,
-								  NULL /* tapleaf_hashes */, 0 /* tapleaf_hashes_len */,
-								  fingerprint, sizeof(fingerprint),
-								  empty_path, ARRAY_SIZE(empty_path));
-		assert(wally_err == WALLY_OK);
-	} else {
-		wally_err = wally_psbt_input_keypath_add(&psbt->inputs[in],
-								  pk_der, sizeof(pk_der),
-								  fingerprint, sizeof(fingerprint),
-								  empty_path, ARRAY_SIZE(empty_path));
-		assert(wally_err == WALLY_OK);
-	}
+	wally_err = wally_psbt_input_add_keypath_item(&psbt->inputs[in],
+						      pk_der, sizeof(pk_der),
+						      fingerprint, sizeof(fingerprint),
+						      empty_path, ARRAY_SIZE(empty_path));
+	assert(wally_err == WALLY_OK);
 	tal_wally_end(psbt);
 }
 
@@ -363,57 +286,6 @@ bool psbt_input_set_signature(struct wally_psbt *psbt, size_t in,
 	return ok;
 }
 
-bool psbt_input_have_signature(const struct wally_psbt *psbt,
-			       size_t in,
-			       const struct pubkey *pubkey,
-			       bool *signature_found)
-{
-	u8 pk_der[PUBKEY_CMPR_LEN];
-	size_t index_plus_one;
-	bool ok;
-
-	assert(in < psbt->num_inputs);
-
-	pubkey_to_der(pk_der, pubkey);
-
-	ok = wally_psbt_input_find_signature(&psbt->inputs[in], pk_der,
-					     sizeof(pk_der),
-					     &index_plus_one) == WALLY_OK;
-	if (ok)
-		*signature_found = index_plus_one > 0;
-	return ok;
-}
-
-bool psbt_input_get_ecdsa_sig(const tal_t *ctx,
-			      const struct wally_psbt *psbt,
-			      size_t in,
-			      const struct pubkey *pubkey,
-			      struct bitcoin_signature **sig)
-{
-	u8 pk_der[PUBKEY_CMPR_LEN];
-	size_t index_plus_one;
-	struct wally_map_item *item;
-	bool ok;
-
-	assert(in < psbt->num_inputs);
-
-	pubkey_to_der(pk_der, pubkey);
-	*sig = NULL;
-
-	ok = wally_psbt_input_find_signature(&psbt->inputs[in], pk_der,
-					     sizeof(pk_der),
-					     &index_plus_one) == WALLY_OK;
-	if (ok) {
-		item = &psbt->inputs[in].signatures.items[index_plus_one - 1];
-		*sig = tal(ctx, struct bitcoin_signature);
-		if (!signature_from_der(item->value, item->value_len, *sig)) {
-			*sig = tal_free(*sig);
-			return false;
-		}
-	}
-	return ok;
-}
-
 void psbt_input_set_wit_utxo(struct wally_psbt *psbt, size_t in,
 			     const u8 *scriptPubkey, struct amount_sat amt)
 {
@@ -462,15 +334,6 @@ void psbt_input_set_utxo(struct wally_psbt *psbt, size_t in,
 	assert(wally_err == WALLY_OK);
 }
 
-void psbt_input_set_outpoint(struct wally_psbt *psbt, size_t in,
-			     struct bitcoin_outpoint outpoint)
-{
-	assert(in < psbt->num_inputs);
-	psbt->inputs[in].index = outpoint.n;
-	memcpy(psbt->inputs[in].txhash, &outpoint.txid,
-	       sizeof(struct bitcoin_txid));
-}
-
 void psbt_input_set_witscript(struct wally_psbt *psbt, size_t in, const u8 *wscript)
 {
 	int wally_err;
@@ -489,7 +352,7 @@ void psbt_elements_input_set_asset(struct wally_psbt *psbt, size_t in,
 	tal_wally_start();
 
 	if (asset->value > 0)
-		if (wally_psbt_input_set_amount(&psbt->inputs[in],
+		if (wally_psbt_input_set_value(&psbt->inputs[in],
 					       asset->value) != WALLY_OK)
 			abort();
 
@@ -503,6 +366,7 @@ void psbt_elements_input_set_asset(struct wally_psbt *psbt, size_t in,
 
 void psbt_elements_normalize_fees(struct wally_psbt *psbt)
 {
+	struct amount_asset asset;
 	size_t fee_output_idx = psbt->num_outputs;
 
 	if (!is_elements(chainparams))
@@ -510,15 +374,15 @@ void psbt_elements_normalize_fees(struct wally_psbt *psbt)
 
 	/* Elements requires that every input value is accounted for,
 	 * including the fees */
-	struct amount_sat total_fee = AMOUNT_SAT(0), val;
+	struct amount_sat total_in = AMOUNT_SAT(0), val;
 	for (size_t i = 0; i < psbt->num_inputs; i++) {
 		val = psbt_input_get_amount(psbt, i);
-		if (!amount_sat_add(&total_fee, total_fee, val))
+		if (!amount_sat_add(&total_in, total_in, val))
 			return;
 	}
 	for (size_t i = 0; i < psbt->num_outputs; i++) {
-		struct amount_asset output_amount = wally_psbt_output_get_amount(&psbt->outputs[i]);
-		if (elements_psbt_output_is_fee(psbt, i)) {
+		asset = wally_tx_output_get_amount(&psbt->tx->outputs[i]);
+		if (elements_wtx_output_is_fee(psbt->tx, i)) {
 			if (fee_output_idx == psbt->num_outputs) {
 				fee_output_idx = i;
 				continue;
@@ -528,46 +392,40 @@ void psbt_elements_normalize_fees(struct wally_psbt *psbt)
 			psbt_rm_output(psbt, i--);
 			continue;
 		}
-		if (!amount_asset_is_main(&output_amount))
+		if (!amount_asset_is_main(&asset))
 			continue;
 
-		if (!amount_sat_sub(&total_fee, total_fee,
-				    amount_asset_to_sat(&output_amount)))
+		if (!amount_sat_sub(&total_in, total_in,
+				    amount_asset_to_sat(&asset)))
 			return;
 	}
 
-	if (amount_sat_eq(total_fee, AMOUNT_SAT(0)))
+	if (amount_sat_eq(total_in, AMOUNT_SAT(0)))
 		return;
 
 	/* We need to add a fee output */
 	if (fee_output_idx == psbt->num_outputs) {
-		psbt_append_output(psbt, NULL, total_fee);
+		psbt_append_output(psbt, NULL, total_in);
 	} else {
-		int ret;
-		u64 sats = total_fee.satoshis; /* Raw: wally API */
-		struct wally_psbt_output *out = &psbt->outputs[fee_output_idx];
-		ret = wally_psbt_output_set_amount(out, sats);
-		assert(ret == WALLY_OK);
+		u64 sats = total_in.satoshis; /* Raw: wally API */
+		struct wally_tx_output *out = &psbt->tx->outputs[fee_output_idx];
+		if (wally_tx_confidential_value_from_satoshi(
+			sats, out->value, out->value_len) != WALLY_OK)
+			return;
 	}
 }
 
-void wally_psbt_input_get_txid(const struct wally_psbt_input *in,
-			       struct bitcoin_txid *txid)
-{
-	CROSS_TYPE_ASSIGNMENT(txid, &in->txhash);
-}
-
 bool psbt_has_input(const struct wally_psbt *psbt,
 		    const struct bitcoin_outpoint *outpoint)
 {
 	for (size_t i = 0; i < psbt->num_inputs; i++) {
 		struct bitcoin_txid in_txid;
-		const struct wally_psbt_input *in = &psbt->inputs[i];
+		struct wally_tx_input *in = &psbt->tx->inputs[i];
 
 		if (outpoint->n != in->index)
 			continue;
 
-		wally_psbt_input_get_txid(in, &in_txid);
+		wally_tx_input_get_txid(in, &in_txid);
 		if (bitcoin_txid_eq(&outpoint->txid, &in_txid))
 			return true;
 	}
@@ -585,7 +443,7 @@ struct amount_sat psbt_input_get_amount(const struct wally_psbt *psbt,
 		assert(amount_asset_is_main(&amt_asset));
 		val = amount_asset_to_sat(&amt_asset);
 	} else if (psbt->inputs[in].utxo) {
-		int idx = psbt->inputs[in].index;
+		int idx = psbt->tx->inputs[in].index;
 		struct wally_tx *prev_tx = psbt->inputs[in].utxo;
 		val = amount_sat(prev_tx->outputs[idx].satoshi);
 	} else
@@ -594,45 +452,16 @@ struct amount_sat psbt_input_get_amount(const struct wally_psbt *psbt,
 	return val;
 }
 
-size_t psbt_input_get_weight(const struct wally_psbt *psbt,
-			     size_t in)
-{
-	size_t weight;
-	const struct wally_map_item *redeem_script;
-
-	redeem_script = wally_map_get_integer(&psbt->inputs[in].psbt_fields, /* PSBT_IN_REDEEM_SCRIPT */ 0x04);
-
-	/* txid + txout + sequence */
-	weight = (32 + 4 + 4) * 4;
-	if (redeem_script) {
-		weight +=
-			(redeem_script->value_len +
-				varint_size(redeem_script->value_len)) * 4;
-	} else {
-		/* zero scriptSig length */
-		weight += varint_size(0) * 4;
-	}
-
-	return weight;
-}
-
 struct amount_sat psbt_output_get_amount(const struct wally_psbt *psbt,
 					 size_t out)
 {
 	struct amount_asset asset;
 	assert(out < psbt->num_outputs);
-	asset = wally_psbt_output_get_amount(&psbt->outputs[out]);
+	asset = wally_tx_output_get_amount(&psbt->tx->outputs[out]);
 	assert(amount_asset_is_main(&asset));
 	return amount_asset_to_sat(&asset);
 }
 
-size_t psbt_output_get_weight(const struct wally_psbt *psbt,
-			      size_t outnum)
-{
-	return (8 /* amount*/ + varint_size(psbt->outputs[outnum].script_len) +
-		psbt->outputs[outnum].script_len) * 4;
-}
-
 static void add(u8 **key, const void *mem, size_t len)
 {
 	size_t oldlen = tal_count(*key);
@@ -645,12 +474,12 @@ static void add_type(u8 **key, const u8 num)
 	add(key, &num, 1);
 }
 
-void add_varint(u8 **arr, size_t val)
+static void add_varint(u8 **key, size_t val)
 {
 	u8 vt[VARINT_MAX_LEN];
 	size_t vtlen;
 	vtlen = varint_put(vt, val);
-	tal_expand(arr, vt, vtlen);
+	add(key, vt, vtlen);
 }
 
 #define LIGHTNING_PROPRIETARY_PREFIX "lightning"
@@ -667,7 +496,7 @@ u8 *psbt_make_key(const tal_t *ctx, u8 key_subtype, const u8 *key_data)
 	 *** <tt><data></tt>
 	 */
 	u8 *key = tal_arr(ctx, u8, 0);
-	add_type(&key, WALLY_PSBT_PROPRIETARY_TYPE);
+	add_type(&key, PSBT_PROPRIETARY_TYPE);
 	add_varint(&key, strlen(LIGHTNING_PROPRIETARY_PREFIX));
 	add(&key, LIGHTNING_PROPRIETARY_PREFIX,
 	    strlen(LIGHTNING_PROPRIETARY_PREFIX));
@@ -677,18 +506,37 @@ u8 *psbt_make_key(const tal_t *ctx, u8 key_subtype, const u8 *key_data)
 	return key;
 }
 
-static void map_replace(const tal_t *ctx,
-			struct wally_map *map,
-			const u8 *key,
-			const void *value,
-			size_t value_len)
+static bool wally_map_set_unknown(const tal_t *ctx,
+				  struct wally_map *map,
+				  const u8 *key,
+				  const void *value,
+				  size_t value_len)
 {
-	const unsigned char *checked_value = memcheck(value, value_len);
-	tal_wally_start();
-	if (wally_map_replace(map, key, tal_bytelen(key),
-			      checked_value, value_len) != WALLY_OK)
-		abort();
-	tal_wally_end(ctx);
+	size_t exists_at;
+	struct wally_map_item *item;
+
+	assert(value_len != 0);
+	if (wally_map_find(map, key, tal_bytelen(key), &exists_at) != WALLY_OK)
+		return false;
+
+	/* If not exists, add */
+	if (exists_at == 0) {
+		bool ok;
+		tal_wally_start();
+		ok = wally_map_add(map, key, tal_bytelen(key),
+			      (unsigned char *) memcheck(value, value_len), value_len)
+			== WALLY_OK;
+		tal_wally_end(ctx);
+		return ok;
+	}
+
+	/* Already in map, update entry */
+	item = &map->items[exists_at - 1];
+	tal_resize(&item->value, value_len);
+	memcpy(item->value, memcheck(value, value_len), value_len);
+	item->value_len = value_len;
+
+	return true;
 }
 
 void psbt_input_set_unknown(const tal_t *ctx,
@@ -697,33 +545,39 @@ void psbt_input_set_unknown(const tal_t *ctx,
 			    const void *value,
 			    size_t value_len)
 {
-	map_replace(ctx, &in->unknowns, key, value, value_len);
+	if (!wally_map_set_unknown(ctx, &in->unknowns, key, value, value_len))
+		abort();
+}
+
+static void *psbt_get_unknown(const struct wally_map *map,
+			      const u8 *key,
+			      size_t *val_len)
+{
+	size_t index;
+
+	if (wally_map_find(map, key, tal_bytelen(key), &index) != WALLY_OK)
+		return NULL;
+
+	/* Zero: item not found. */
+	if (index == 0)
+		return NULL;
+
+	/* ++: item is at this index minus 1 */
+	*val_len = map->items[index - 1].value_len;
+	return map->items[index - 1].value;
 }
 
 void *psbt_get_lightning(const struct wally_map *map,
 			 const u8 proprietary_type,
 			 size_t *val_len)
 {
+	void *res;
 	u8 *key = psbt_make_key(NULL, proprietary_type, NULL);
-	const struct wally_map_item *item;
-	item = wally_map_get(map, key, tal_bytelen(key));
+	res = psbt_get_unknown(map, key, val_len);
 	tal_free(key);
-	if (!item)
-		return NULL;
-	*val_len = item->value_len;
-	return item->value;
+	return res;
 }
 
-void psbt_set_lightning(const tal_t *ctx,
-			struct wally_map *map,
-			const u8 proprietary_type,
-			const void *value,
-			size_t val_len)
-{
-	u8 *key = psbt_make_key(NULL, proprietary_type, NULL);
-	map_replace(ctx, map, key, value, val_len);
-	tal_free(key);
-}
 
 void psbt_output_set_unknown(const tal_t *ctx,
 			     struct wally_psbt_output *out,
@@ -731,7 +585,8 @@ void psbt_output_set_unknown(const tal_t *ctx,
 			     const void *value,
 			     size_t value_len)
 {
-	map_replace(ctx, &out->unknowns, key, value, value_len);
+	if (!wally_map_set_unknown(ctx, &out->unknowns, key, value, value_len))
+		abort();
 }
 
 /* Use the destructor to free the wally_tx */
@@ -747,24 +602,16 @@ bool psbt_finalize(struct wally_psbt *psbt)
 	tal_wally_start();
 
 	/* Wally doesn't know how to finalize P2WSH; this happens with
-	 * option_anchor_outputs, and finalizing those two cases is trivial. */
+	 * option_anchor_outputs, and finalizing is trivial. */
 	/* FIXME: miniscript! miniscript! miniscript! */
 	for (size_t i = 0; i < psbt->num_inputs; i++) {
 		struct wally_psbt_input *input = &psbt->inputs[i];
 		struct wally_tx_witness_stack *stack;
-		const struct wally_map_item *iws;
 
-		iws = wally_map_get_integer(&input->psbt_fields, /* PSBT_IN_WITNESS_SCRIPT */ 0x05);
-		if (!iws)
+		if (!is_anchor_witness_script(input->witness_script,
+					      input->witness_script_len))
 			continue;
 
-		if (!is_to_remote_anchored_witness_script(iws->value,
-							  iws->value_len)
-		    && !is_anchor_witness_script(iws->value,
-						 iws->value_len)) {
-			continue;
-		}
-
 		if (input->signatures.num_items != 1)
 			continue;
 
@@ -782,31 +629,17 @@ bool psbt_finalize(struct wally_psbt *psbt)
 		 *
 		 *    <remote_sig>
 		 */
-		/* BOLT #3:
-		 * #### `to_local_anchor` and `to_remote_anchor` Output (option_anchors)
-		 *...
-		 *    <local_funding_pubkey/remote_funding_pubkey> OP_CHECKSIG OP_IFDUP
-		 *    OP_NOTIF
-		 *        OP_16 OP_CHECKSEQUENCEVERIFY
-		 *    OP_ENDIF
-		 *...
-		 * Spending of the output requires the following witness:
-		 *     <local_sig/remote_sig>
-		 */
-
-		/* i.e. in both cases, this is the same thing */
 		wally_tx_witness_stack_init_alloc(2, &stack);
 		wally_tx_witness_stack_add(stack,
 					   input->signatures.items[0].value,
 					   input->signatures.items[0].value_len);
 		wally_tx_witness_stack_add(stack,
-					   iws->value,
-					   iws->value_len);
+					   input->witness_script,
+					   input->witness_script_len);
 		wally_psbt_input_set_final_witness(input, stack);
-		wally_tx_witness_stack_free(stack);
 	}
 
-	ok = (wally_psbt_finalize(psbt, 0 /* flags */) == WALLY_OK);
+	ok = (wally_psbt_finalize(psbt) == WALLY_OK);
 	tal_wally_end(psbt);
 
 	return ok && psbt_is_finalized(psbt);
@@ -820,7 +653,7 @@ struct wally_tx *psbt_final_tx(const tal_t *ctx, const struct wally_psbt *psbt)
 		return NULL;
 
 	tal_wally_start();
-	if (wally_psbt_extract(psbt, /* flags */ 0, &wtx) == WALLY_OK)
+	if (wally_psbt_extract(psbt, &wtx) == WALLY_OK)
 		tal_add_destructor(wtx, wally_tx_destroy);
 	else
 		wtx = NULL;
@@ -834,9 +667,10 @@ struct wally_psbt *psbt_from_b64(const tal_t *ctx,
 				 size_t b64len)
 {
 	struct wally_psbt *psbt;
+	char *str = tal_strndup(tmpctx, b64, b64len);
 
 	tal_wally_start();
-	if (wally_psbt_from_base64_n(b64, b64len, /* flags */ 0, &psbt) == WALLY_OK)
+	if (wally_psbt_from_base64(str, &psbt) == WALLY_OK)
 		tal_add_destructor(psbt, psbt_destroy);
 	else
 		psbt = NULL;
@@ -845,7 +679,7 @@ struct wally_psbt *psbt_from_b64(const tal_t *ctx,
 	return psbt;
 }
 
-char *fmt_wally_psbt(const tal_t *ctx, const struct wally_psbt *psbt)
+char *psbt_to_b64(const tal_t *ctx, const struct wally_psbt *psbt)
 {
 	char *serialized_psbt;
 	int ret;
@@ -857,6 +691,7 @@ char *fmt_wally_psbt(const tal_t *ctx, const struct wally_psbt *psbt)
 
 	return serialized_psbt;
 }
+REGISTER_TYPE_TO_STRING(wally_psbt, psbt_to_b64);
 
 const u8 *psbt_get_bytes(const tal_t *ctx, const struct wally_psbt *psbt,
 			 size_t *bytes_written)
@@ -869,9 +704,7 @@ const u8 *psbt_get_bytes(const tal_t *ctx, const struct wally_psbt *psbt,
 		return NULL;
 	}
 
-	if (wally_psbt_get_length(psbt, 0, &len) != WALLY_OK) {
-		abort();
-	}
+	wally_psbt_get_length(psbt, 0, &len);
 	bytes = tal_arr(ctx, u8, len);
 
 	if (wally_psbt_to_bytes(psbt, 0, bytes, len, bytes_written) != WALLY_OK ||
@@ -882,19 +715,13 @@ const u8 *psbt_get_bytes(const tal_t *ctx, const struct wally_psbt *psbt,
 	return bytes;
 }
 
-bool validate_psbt(const struct wally_psbt *psbt)
-{
-	size_t len;
-	return wally_psbt_get_length(psbt, 0, &len) == WALLY_OK;
-}
-
 struct wally_psbt *psbt_from_bytes(const tal_t *ctx, const u8 *bytes,
 				   size_t byte_len)
 {
 	struct wally_psbt *psbt;
 
 	tal_wally_start();
-	if (wally_psbt_from_bytes(bytes, byte_len, /* flags */ 0, &psbt) == WALLY_OK)
+	if (wally_psbt_from_bytes(bytes, byte_len, &psbt) == WALLY_OK)
 		tal_add_destructor(psbt, psbt_destroy);
 	else
 		psbt = NULL;
@@ -905,24 +732,12 @@ struct wally_psbt *psbt_from_bytes(const tal_t *ctx, const u8 *bytes,
 
 void towire_wally_psbt(u8 **pptr, const struct wally_psbt *psbt)
 {
-	struct wally_psbt *psbt_copy;
-
 	/* Let's include the PSBT bytes */
 	size_t bytes_written;
-	const u8 *psbt_bytes = psbt_get_bytes(NULL, psbt, &bytes_written);
-
-	/* When sending to other processes, set to v0 for compat */
-	psbt_copy = psbt_from_bytes(NULL, psbt_bytes, bytes_written);
-	tal_free(psbt_bytes);
-    if (!is_elements(chainparams))
-		psbt_set_version(psbt_copy, 0);
-
-	const u8 *psbt_bytes_copy = psbt_get_bytes(NULL, psbt_copy, &bytes_written);
-
+	const u8 *pbt_bytes = psbt_get_bytes(NULL, psbt, &bytes_written);
 	towire_u32(pptr, bytes_written);
-	towire_u8_array(pptr, psbt_bytes_copy, bytes_written);
-	tal_free(psbt_bytes_copy);
-	tal_free(psbt_copy);
+	towire_u8_array(pptr, pbt_bytes, bytes_written);
+	tal_free(pbt_bytes);
 }
 
 struct wally_psbt *fromwire_wally_psbt(const tal_t *ctx,
@@ -941,29 +756,53 @@ struct wally_psbt *fromwire_wally_psbt(const tal_t *ctx,
 	if (!psbt)
 		return fromwire_fail(cursor, max);
 
-	/* Internally we always operate on v2 */
-	psbt_set_version(psbt, 2);
+#if DEVELOPER
+	/* Re-marshall for sanity check! */
+	u8 *tmpbuf = tal_arr(NULL, u8, psbt_byte_len);
+	size_t written;
+	if (wally_psbt_to_bytes(psbt, 0, tmpbuf, psbt_byte_len, &written) != WALLY_OK) {
+		tal_free(tmpbuf);
+		tal_free(psbt);
+		return fromwire_fail(cursor, max);
+	}
+	tal_free(tmpbuf);
+#endif
 
 	return psbt;
 }
 
+/* This only works on a non-final psbt because we're ALL SEGWIT! */
 void psbt_txid(const tal_t *ctx,
-	       const struct wally_psbt *psbt,
-		   struct bitcoin_txid *txid,
+	       const struct wally_psbt *psbt, struct bitcoin_txid *txid,
 	       struct wally_tx **wtx)
 {
 	struct wally_tx *tx;
-	int wally_err;
-	assert(psbt->version == 2);
 
-	/* We rely on wally extractor to fill out all txid-related fields including scriptSigs */
+	/* You can *almost* take txid of global tx.  But @niftynei thought
+	 * about this far more than me and pointed out that P2SH
+	 * inputs would not be represented, so here we go. */
 	tal_wally_start();
-	wally_err = wally_psbt_extract(psbt, WALLY_PSBT_EXTRACT_NON_FINAL, &tx);
-	assert(wally_err == WALLY_OK);
-	wally_err = wally_tx_get_txid(tx, txid->shad.sha.u.u8, sizeof(txid->shad.sha.u.u8));
-	assert(wally_err == WALLY_OK);
-	tal_wally_end(ctx);
+	wally_tx_clone_alloc(psbt->tx, 0, &tx);
 
+	for (size_t i = 0; i < tx->num_inputs; i++) {
+		if (psbt->inputs[i].final_scriptsig) {
+			wally_tx_set_input_script(tx, i,
+						  psbt->inputs[i].final_scriptsig,
+						  psbt->inputs[i].final_scriptsig_len);
+		} else if (psbt->inputs[i].redeem_script) {
+			u8 *script;
+
+			/* P2SH requires push of the redeemscript, from libwally src */
+			script = tal_arr(tmpctx, u8, 0);
+			script_push_bytes(&script,
+					  psbt->inputs[i].redeem_script,
+					  psbt->inputs[i].redeem_script_len);
+			wally_tx_set_input_script(tx, i, script, tal_bytelen(script));
+		}
+	}
+	tal_wally_end_onto(ctx, tx, struct wally_tx);
+
+	wally_txid(tx, txid);
 	if (wtx)
 		*wtx = tx;
 	else
@@ -984,9 +823,9 @@ struct amount_sat psbt_compute_fee(const struct wally_psbt *psbt)
 	}
 
 	for (size_t i = 0; i < psbt->num_outputs; i++) {
-		asset = wally_psbt_output_get_amount(&psbt->outputs[i]);
+		asset = wally_tx_output_get_amount(&psbt->tx->outputs[i]);
 		if (!amount_asset_is_main(&asset)
-		    || elements_psbt_output_is_fee(psbt, i))
+		    || elements_wtx_output_is_fee(psbt->tx, i))
 			continue;
 
 		ok = amount_sat_sub(&fee, fee, amount_asset_to_sat(&asset));
@@ -996,89 +835,3 @@ struct amount_sat psbt_compute_fee(const struct wally_psbt *psbt)
 
 	return fee;
 }
-
-bool wally_psbt_input_spends(const struct wally_psbt_input *input,
-			     const struct bitcoin_outpoint *outpoint)
-{
-	/* Useful, as tx_part can have some NULL inputs */
-	if (!input)
-		return false;
-	if (input->index != outpoint->n)
-		return false;
-	return CROSS_TYPE_EQ(&outpoint->txid, &input->txhash);
-}
-
-void wally_psbt_input_get_outpoint(const struct wally_psbt_input *in,
-				   struct bitcoin_outpoint *outpoint)
-{
-	CROSS_TYPE_ASSIGNMENT(&outpoint->txid, &in->txhash);
-	outpoint->n = in->index;
-}
-
-const u8 *wally_psbt_output_get_script(const tal_t *ctx,
-                     const struct wally_psbt_output *output)
-{
-    if (output->script == NULL) {
-        /* This can happen for coinbase transactions, pegin
-         * transactions, and elements fee outputs */
-        return NULL;
-    }
-
-    return tal_dup_arr(ctx, u8, output->script, output->script_len, 0);
-}
-
-/* FIXME(cdecker) Make the caller pass in a reference to amount_asset, and
- * return false if unintelligible/encrypted. (WARN UNUSED). */
-struct amount_asset
-wally_psbt_output_get_amount(const struct wally_psbt_output *output)
-{
-    struct amount_asset amount;
-	size_t asset_out;
-
-    if (chainparams->is_elements) {
-		if (wally_psbt_output_get_asset(output, amount.asset + 1, sizeof(amount.asset) - 1, &asset_out) != WALLY_OK) {
-			amount.value = 0;
-			return amount;
-		}
-		assert(asset_out == 32);
-		amount.asset[0] = 0x01; /* explicit */
-        /* We currently only support explicit value
-         * asset tags, others are confidential, so
-         * don't even try to assign a value to it. */
-        if (output->has_amount == true) {
-            amount.value = output->amount;
-        } else {
-            amount.value = 0;
-        }
-    } else {
-        /* Do not assign amount.asset, we should never touch it in
-         * non-elements scenarios. */
-		if (output->has_amount) {
-        	amount.value = output->amount;
-		} else {
-			abort();
-		}
-    }
-
-    return amount;
-}
-
-bool elements_psbt_output_is_fee(const struct wally_psbt *psbt, size_t outnum)
-{
-    assert(outnum < psbt->num_outputs);
-    return chainparams->is_elements &&
-        psbt->outputs[outnum].script_len == 0;
-}
-
-bool psbt_set_version(struct wally_psbt *psbt, u32 version)
-{
-	bool ok;
-
-	tal_wally_start();
-	ok = wally_psbt_set_version(psbt, 0, version) == WALLY_OK;
-	if (ok && version == 2) {
-		ok &= wally_psbt_set_tx_modifiable_flags(psbt, WALLY_PSBT_TXMOD_INPUTS | WALLY_PSBT_TXMOD_OUTPUTS) == WALLY_OK;
-	}
-	tal_wally_end(psbt);
-	return ok;
-}

bitcoin/psbt.h

@@ -17,11 +17,6 @@ struct bitcoin_signature;
 struct bitcoin_txid;
 struct pubkey;
 
-
-/* Utility we need for psbt stuffs;
- * add the varint onto the given array */
-void add_varint(u8 **arr, size_t val);
-
 /**
  * create_psbt - Create a new psbt object
  *
@@ -34,7 +29,7 @@ struct wally_psbt *create_psbt(const tal_t *ctx, size_t num_inputs, size_t num_o
 
 /*
  * new_psbt - Create a PSBT, using the passed in tx
- * 	      as the locktime/inputs/output psbt fields
+ * 	      as the global_tx
  *
  * @ctx - allocation context
  * @wtx - global_tx starter kit
@@ -48,29 +43,7 @@ struct wally_psbt *new_psbt(const tal_t *ctx,
  * @ctx - allocation context
  * @psbt - psbt to be cloned
  */
-struct wally_psbt *clone_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
-
-/**
- * combine_psbt - Combine two PSBT into a cloned copy
- *
- * @ctx - allocation context
- * @psbt0 - one psbt
- * @psbt1 - other psbt
- */
-struct wally_psbt *combine_psbt(const tal_t *ctx,
-				const struct wally_psbt *psbt0,
-				const struct wally_psbt *psbt1);
-
-/**
- * audit_psbt - Audit the memory structure of the PSBT.
- *
- * This checks all known memory allocations in the PSBT and asserts that they
- * are all allocated with 'ctx' being it's parent.
- *
- * ctx - the ctx all memory *should* be attached to
- * psbt - the PSBT to audit.
- * */
-void audit_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
+struct wally_psbt *clone_psbt(const tal_t *ctx, struct wally_psbt *psbt);
 
 /**
  * psbt_is_finalized - Check if tx is ready to be extracted
@@ -130,7 +103,7 @@ struct wally_tx *psbt_final_tx(const tal_t *ctx, const struct wally_psbt *psbt);
 u8 *psbt_make_key(const tal_t *ctx, u8 key_subtype, const u8 *key_data);
 
 struct wally_psbt_input *psbt_add_input(struct wally_psbt *psbt,
-					const struct wally_tx_input *input,
+					struct wally_tx_input *input,
 					size_t insert_at);
 
 /* One stop shop for adding an input + metadata to a PSBT */
@@ -149,9 +122,6 @@ void psbt_input_set_wit_utxo(struct wally_psbt *psbt, size_t in,
 void psbt_input_set_utxo(struct wally_psbt *psbt, size_t in,
 			 const struct wally_tx *prev_tx);
 
-void psbt_input_set_outpoint(struct wally_psbt *psbt, size_t in,
-			     struct bitcoin_outpoint outpoint);
-
 /* psbt_elements_input_set_asset - Set the asset/value fields for an
  * 				   Elements PSBT (PSET, technically */
 void psbt_elements_input_set_asset(struct wally_psbt *psbt, size_t in,
@@ -183,28 +153,12 @@ void psbt_rm_output(struct wally_psbt *psbt,
 		    size_t remove_at);
 
 void psbt_input_add_pubkey(struct wally_psbt *psbt, size_t in,
-			   const struct pubkey *pubkey, bool is_taproot);
+			   const struct pubkey *pubkey);
 
 WARN_UNUSED_RESULT bool psbt_input_set_signature(struct wally_psbt *psbt, size_t in,
 						 const struct pubkey *pubkey,
 						 const struct bitcoin_signature *sig);
 
-/* Returns false on error. On success, *signature_found is set to true if the
- * input has a signature present for `pubkey` and false if if one was not found.
- * Only signature presence is checked, it is not validated. */
-WARN_UNUSED_RESULT bool psbt_input_have_signature(const struct wally_psbt *psbt,
-						  size_t in,
-						  const struct pubkey *pubkey,
-						  bool *signature_found);
-
-/* Returns false on error. On success *sig is set to the signature otherwise
- * *sig is set to NULL. */
-WARN_UNUSED_RESULT bool psbt_input_get_ecdsa_sig(const tal_t *ctx,
-						 const struct wally_psbt *psbt,
-						 size_t in,
-						 const struct pubkey *pubkey,
-						 struct bitcoin_signature **sig);
-
 void psbt_input_set_witscript(struct wally_psbt *psbt, size_t in, const u8 *wscript);
 
 /* psbt_input_set_unknown - Set the given Key-Value in the psbt's input keymap
@@ -231,19 +185,6 @@ void *psbt_get_lightning(const struct wally_map *map,
 			 const u8 proprietary_type,
 			 size_t *val_len);
 
-/* psbt_set_lightning - Set a propreitary lightning value on the given map
- *
- * @map - map of unknowns to set the value
- * @proprietary_type - type no. to set
- * @value - the value to be set
- * @val_len - length of value
- */
-void psbt_set_lightning(const tal_t *ctx,
-			struct wally_map *map,
-			const u8 proprietary_type,
-			const void *value,
-			size_t val_len);
-
 /* psbt_output_set_unknown - Set the given Key-Value in the psbt's output keymap
  *
  * @ctx - tal context for allocations
@@ -265,10 +206,6 @@ void psbt_output_set_unknown(const tal_t *ctx,
 struct amount_sat psbt_input_get_amount(const struct wally_psbt *psbt,
 					size_t in);
 
-/* psbt_input_get_weight - Calculate the tx weight for input index `in` */
-size_t psbt_input_get_weight(const struct wally_psbt *psbt,
-			     size_t in);
-
 /* psbt_output_get_amount - Returns the value of this output
  *
  * @psbt - psbt
@@ -277,10 +214,6 @@ size_t psbt_input_get_weight(const struct wally_psbt *psbt,
 struct amount_sat psbt_output_get_amount(const struct wally_psbt *psbt,
 					 size_t out);
 
-/* psbt_output_get_weight - Calculate the tx weight for output index `outnum` */
-size_t psbt_output_get_weight(const struct wally_psbt *psbt,
-			      size_t outnum);
-
 /* psbt_compute_fee - Returns value of fee for PSBT
  *
  * @psbt -psbt
@@ -295,40 +228,12 @@ struct amount_sat psbt_compute_fee(const struct wally_psbt *psbt);
 bool psbt_has_input(const struct wally_psbt *psbt,
 		    const struct bitcoin_outpoint *outpoint);
 
-/* wally_psbt_input_spends - Returns true if PSBT input spends given outpoint
- *
- * @input - psbt input
- * @outpoint - outpoint
- */
-bool wally_psbt_input_spends(const struct wally_psbt_input *input,
-               const struct bitcoin_outpoint *outpoint);
-
-void wally_psbt_input_get_outpoint(const struct wally_psbt_input *in,
-                 struct bitcoin_outpoint *outpoint);
-
-const u8 *wally_psbt_output_get_script(const tal_t *ctx,
-                     const struct wally_psbt_output *output);
-
-void wally_psbt_input_get_txid(const struct wally_psbt_input *in,
-                 struct bitcoin_txid *txid);
-
-struct amount_asset
-wally_psbt_output_get_amount(const struct wally_psbt_output *output);
-
-/* psbt_set_version - Returns false if there was any issue with the PSBT.
- * Returns true if it was a well-formed PSET and treats it as a no-op
- */
-bool psbt_set_version(struct wally_psbt *psbt, u32 version);
-
-bool elements_psbt_output_is_fee(const struct wally_psbt *psbt, size_t outnum);
-
 struct wally_psbt *psbt_from_b64(const tal_t *ctx,
 				 const char *b64,
 				 size_t b64len);
-char *fmt_wally_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
+char *psbt_to_b64(const tal_t *ctx, const struct wally_psbt *psbt);
 const u8 *psbt_get_bytes(const tal_t *ctx, const struct wally_psbt *psbt,
 			 size_t *bytes_written);
-bool validate_psbt(const struct wally_psbt *psbt);
 struct wally_psbt *psbt_from_bytes(const tal_t *ctx, const u8 *bytes,
 				   size_t byte_len);
 void towire_wally_psbt(u8 **pptr, const struct wally_psbt *psbt);

bitcoin/pubkey.c

@@ -4,7 +4,7 @@
 #include <bitcoin/pubkey.h>
 #include <ccan/mem/mem.h>
 #include <ccan/str/hex/hex.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
 #ifndef SUPERVERBOSE
@@ -62,15 +62,16 @@ bool pubkey_from_hexstr(const char *derstr, size_t slen, struct pubkey *key)
 	return pubkey_from_der(der, dlen, key);
 }
 
-char *fmt_pubkey(const tal_t *ctx, const struct pubkey *key)
+char *pubkey_to_hexstr(const tal_t *ctx, const struct pubkey *key)
 {
 	unsigned char der[PUBKEY_CMPR_LEN];
 
 	pubkey_to_der(der, key);
 	return tal_hexstr(ctx, der, sizeof(der));
 }
+REGISTER_TYPE_TO_STRING(pubkey, pubkey_to_hexstr);
 
-char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key)
+static char *secp256k1_pubkey_to_hexstr(const tal_t *ctx, const secp256k1_pubkey *key)
 {
 	unsigned char der[PUBKEY_CMPR_LEN];
 	size_t outlen = sizeof(der);
@@ -80,10 +81,11 @@ char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key)
 	assert(outlen == sizeof(der));
 	return tal_hexstr(ctx, der, sizeof(der));
 }
+REGISTER_TYPE_TO_STRING(secp256k1_pubkey, secp256k1_pubkey_to_hexstr);
 
 int pubkey_cmp(const struct pubkey *a, const struct pubkey *b)
 {
-	u8 keya[PUBKEY_CMPR_LEN], keyb[PUBKEY_CMPR_LEN];
+	u8 keya[33], keyb[33];
 	pubkey_to_der(keya, a);
 	pubkey_to_der(keyb, b);
 	return memcmp(keya, keyb, sizeof(keya));
@@ -123,3 +125,37 @@ void towire_pubkey(u8 **pptr, const struct pubkey *pubkey)
 
 	towire(pptr, output, outputlen);
 }
+
+void fromwire_point32(const u8 **cursor, size_t *max, struct point32 *point32)
+{
+	u8 raw[32];
+
+	if (!fromwire(cursor, max, raw, sizeof(raw)))
+		return;
+
+	if (secp256k1_xonly_pubkey_parse(secp256k1_ctx,
+					 &point32->pubkey,
+					 raw) != 1) {
+		SUPERVERBOSE("not a valid point");
+		fromwire_fail(cursor, max);
+	}
+}
+
+void towire_point32(u8 **pptr, const struct point32 *point32)
+{
+	u8 output[32];
+
+	secp256k1_xonly_pubkey_serialize(secp256k1_ctx, output,
+					 &point32->pubkey);
+	towire(pptr, output, sizeof(output));
+}
+
+static char *point32_to_hexstr(const tal_t *ctx, const struct point32 *point32)
+{
+	u8 output[32];
+
+	secp256k1_xonly_pubkey_serialize(secp256k1_ctx, output,
+					 &point32->pubkey);
+	return tal_hexstr(ctx, output, sizeof(output));
+}
+REGISTER_TYPE_TO_STRING(point32, point32_to_hexstr);

bitcoin/pubkey.h

@@ -19,12 +19,18 @@ struct pubkey {
 /* Define pubkey_eq (no padding) */
 STRUCTEQ_DEF(pubkey, 0, pubkey.data);
 
+struct point32 {
+	/* Unpacked pubkey (as used by libsecp256k1 internally) */
+	secp256k1_xonly_pubkey pubkey;
+};
+/* Define pubkey_eq (no padding) */
+STRUCTEQ_DEF(point32, 0, pubkey.data);
+
 /* Convert from hex string of DER (scriptPubKey from validateaddress) */
 bool pubkey_from_hexstr(const char *derstr, size_t derlen, struct pubkey *key);
 
 /* Convert from hex string of DER (scriptPubKey from validateaddress) */
-char *fmt_pubkey(const tal_t *ctx, const struct pubkey *key);
-char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key);
+char *pubkey_to_hexstr(const tal_t *ctx, const struct pubkey *key);
 
 /* Point from secret */
 bool pubkey_from_secret(const struct secret *secret, struct pubkey *key);
@@ -57,4 +63,13 @@ void pubkey_to_hash160(const struct pubkey *pk, struct ripemd160 *hash);
 void towire_pubkey(u8 **pptr, const struct pubkey *pubkey);
 void fromwire_pubkey(const u8 **cursor, size_t *max, struct pubkey *pubkey);
 
+/* FIXME: Old spec uses pubkey32 */
+#define pubkey32 point32
+#define towire_pubkey32 towire_point32
+#define fromwire_pubkey32 fromwire_point32
+
+/* marshal/unmarshal functions */
+void towire_point32(u8 **pptr, const struct point32 *pubkey);
+void fromwire_point32(const u8 **cursor, size_t *max, struct point32 *pubkey);
+
 #endif /* LIGHTNING_BITCOIN_PUBKEY_H */

bitcoin/script.c

@@ -310,69 +310,6 @@ u8 *scriptpubkey_witness_raw(const tal_t *ctx, u8 version,
 	return script;
 }
 
-u8 *scriptpubkey_raw_p2tr(const tal_t *ctx, const struct pubkey *output_pubkey)
-{
-	int ok;
-	secp256k1_xonly_pubkey x_key;
-	unsigned char x_key_bytes[32];
-	u8 *script = tal_arr(ctx, u8, 0);
-
-	add_op(&script, OP_1);
-
-	ok = secp256k1_xonly_pubkey_from_pubkey(secp256k1_ctx,
-		&x_key,
-		/* pk_parity */ NULL,
-		&(output_pubkey->pubkey));
-	assert(ok);
-
-	ok = secp256k1_xonly_pubkey_serialize(secp256k1_ctx,
-		x_key_bytes,
-		&x_key);
-	assert(ok);
-
-	script_push_bytes(&script, x_key_bytes, sizeof(x_key_bytes));
-	assert(tal_count(script) == BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
-	return script;
-}
-
-u8 *scriptpubkey_raw_p2tr_derkey(const tal_t *ctx, const u8 output_der[33])
-{
-	struct pubkey tr_key;
-	if (!pubkey_from_der(output_der, 33, &tr_key)) {
-		abort();
-	}
-	return scriptpubkey_raw_p2tr(ctx, &tr_key);
-}
-
-u8 *scriptpubkey_p2tr(const tal_t *ctx, const struct pubkey *inner_pubkey)
-{
-	unsigned char key_bytes[33];
-	unsigned char tweaked_key_bytes[33];
-	size_t out_len = sizeof(key_bytes);
-	u8 *script = tal_arr(ctx, u8, 0);
-
-	add_op(&script, OP_1);
-
-	secp256k1_ec_pubkey_serialize(secp256k1_ctx, key_bytes, &out_len, &inner_pubkey->pubkey, SECP256K1_EC_COMPRESSED);
-	/* Only commit to inner pubkey in tweak */
-	if (wally_ec_public_key_bip341_tweak(key_bytes, 33, /* merkle_root*/ NULL, 0, 0 /* flags */, tweaked_key_bytes, sizeof(tweaked_key_bytes)) != WALLY_OK)
-		abort();
-
-	/* Cut off the first byte from the serialized compressed key */
-	script_push_bytes(&script, tweaked_key_bytes + 1, sizeof(tweaked_key_bytes) - 1);
-	assert(tal_count(script) == BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
-	return script;
-}
-
-u8 *scriptpubkey_p2tr_derkey(const tal_t *ctx, const u8 inner_der[33])
-{
-	struct pubkey tr_key;
-	if (!pubkey_from_der(inner_der, 33, &tr_key)) {
-		abort();
-	}
-	return scriptpubkey_p2tr(ctx, &tr_key);
-}
-
 /* BOLT #3:
  *
  * #### `to_remote` Output
@@ -392,9 +329,9 @@ u8 *scriptpubkey_p2tr_derkey(const tal_t *ctx, const u8 inner_der[33])
  * <remote_pubkey> OP_CHECKSIGVERIFY MAX(1, lease_end - blockheight) OP_CHECKSEQUENCEVERIFY
  */
 
-u8 *bitcoin_wscript_to_remote_anchored(const tal_t *ctx,
-				       const struct pubkey *remote_key,
-				       u32 csv_lock)
+u8 *anchor_to_remote_redeem(const tal_t *ctx,
+			    const struct pubkey *remote_key,
+			    u32 csv_lock)
 {
 	u8 *script = tal_arr(ctx, u8, 0);
 	add_push_key(&script, remote_key);
@@ -402,11 +339,11 @@ u8 *bitcoin_wscript_to_remote_anchored(const tal_t *ctx,
 	add_number(&script, csv_lock);
 	add_op(&script, OP_CHECKSEQUENCEVERIFY);
 
-	assert(is_to_remote_anchored_witness_script(script, tal_bytelen(script)));
+	assert(is_anchor_witness_script(script, tal_bytelen(script)));
 	return script;
 }
 
-bool is_to_remote_anchored_witness_script(const u8 *script, size_t script_len)
+bool is_anchor_witness_script(const u8 *script, size_t script_len)
 {
 	size_t len = 34 + 1 + 1 + 1;
 	/* With option_will_fund, the pushbytes can be up to 2 bytes more
@@ -476,8 +413,10 @@ u8 *p2wpkh_scriptcode(const tal_t *ctx, const struct pubkey *key)
 	return script;
 }
 
-bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
+bool is_p2pkh(const u8 *script, struct bitcoin_address *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2PKH_LEN)
 		return false;
 	if (script[0] != OP_DUP)
@@ -495,8 +434,10 @@ bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
 	return true;
 }
 
-bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr)
+bool is_p2sh(const u8 *script, struct ripemd160 *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2SH_LEN)
 		return false;
 	if (script[0] != OP_HASH160)
@@ -510,8 +451,10 @@ bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr)
 	return true;
 }
 
-bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr)
+bool is_p2wsh(const u8 *script, struct sha256 *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2WSH_LEN)
 		return false;
 	if (script[0] != OP_0)
@@ -523,8 +466,10 @@ bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr)
 	return true;
 }
 
-bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
+bool is_p2wpkh(const u8 *script, struct bitcoin_address *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN)
 		return false;
 	if (script[0] != OP_0)
@@ -536,34 +481,10 @@ bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr
 	return true;
 }
 
-bool is_p2tr(const u8 *script, size_t script_len, u8 xonly_pubkey[32])
+bool is_known_scripttype(const u8 *script)
 {
-	if (script_len != BITCOIN_SCRIPTPUBKEY_P2TR_LEN)
-		return false;
-	if (script[0] != OP_1)
-		return false;
-	/* x-only pubkey */
-	if (script[1] != OP_PUSHBYTES(32))
-		return false;
-	if (xonly_pubkey)
-		memcpy(xonly_pubkey, script+2, 32);
-	return true;
-}
-
-bool is_known_scripttype(const u8 *script, size_t script_len)
-{
-	return is_p2wpkh(script, script_len, NULL)
-		|| is_p2wsh(script, script_len, NULL)
-		|| is_p2sh(script, script_len, NULL)
-		|| is_p2pkh(script, script_len, NULL)
-		|| is_p2tr(script, script_len, NULL);
-}
-
-bool is_known_segwit_scripttype(const u8 *script, size_t script_len)
-{
-	return is_p2wpkh(script, script_len, NULL)
-		|| is_p2wsh(script, script_len, NULL)
-		|| is_p2tr(script, script_len, NULL);
+	return is_p2wpkh(script, NULL) || is_p2wsh(script, NULL)
+		|| is_p2sh(script, NULL) || is_p2pkh(script, NULL);
 }
 
 u8 **bitcoin_witness_sig_and_element(const tal_t *ctx,
@@ -685,8 +606,7 @@ u8 *bitcoin_wscript_htlc_offer_ripemd160(const tal_t *ctx,
 					 const struct pubkey *remotehtlckey,
 					 const struct ripemd160 *payment_ripemd,
 					 const struct pubkey *revocationkey,
-					 bool option_anchor_outputs,
-					 bool option_anchors_zero_fee_htlc_tx)
+					 bool option_anchor_outputs)
 {
 	u8 *script = tal_arr(ctx, u8, 0);
 	struct ripemd160 ripemd;
@@ -718,7 +638,7 @@ u8 *bitcoin_wscript_htlc_offer_ripemd160(const tal_t *ctx,
 	add_op(&script, OP_EQUALVERIFY);
 	add_op(&script, OP_CHECKSIG);
 	add_op(&script, OP_ENDIF);
-	if (option_anchor_outputs || option_anchors_zero_fee_htlc_tx) {
+	if (option_anchor_outputs) {
 		add_number(&script, 1);
 		add_op(&script, OP_CHECKSEQUENCEVERIFY);
 		add_op(&script, OP_DROP);
@@ -733,8 +653,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
 			       const struct pubkey *remotehtlckey,
 			       const struct sha256 *payment_hash,
 			       const struct pubkey *revocationkey,
-			       bool option_anchor_outputs,
-			       bool option_anchors_zero_fee_htlc_tx)
+			       bool option_anchor_outputs)
 {
 	struct ripemd160 ripemd;
 
@@ -742,8 +661,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
 	return bitcoin_wscript_htlc_offer_ripemd160(ctx, localhtlckey,
 						    remotehtlckey,
 						    &ripemd, revocationkey,
-						    option_anchor_outputs,
-						    option_anchors_zero_fee_htlc_tx);
+						    option_anchor_outputs);
 }
 
 /* BOLT #3:
@@ -799,8 +717,7 @@ u8 *bitcoin_wscript_htlc_receive_ripemd(const tal_t *ctx,
 					const struct pubkey *remotehtlckey,
 					const struct ripemd160 *payment_ripemd,
 					const struct pubkey *revocationkey,
-					bool option_anchor_outputs,
-					bool option_anchors_zero_fee_htlc_tx)
+					bool option_anchor_outputs)
 {
 	u8 *script = tal_arr(ctx, u8, 0);
 	struct ripemd160 ripemd;
@@ -835,7 +752,7 @@ u8 *bitcoin_wscript_htlc_receive_ripemd(const tal_t *ctx,
 	add_op(&script, OP_DROP);
 	add_op(&script, OP_CHECKSIG);
 	add_op(&script, OP_ENDIF);
-	if (option_anchor_outputs || option_anchors_zero_fee_htlc_tx) {
+	if (option_anchor_outputs) {
 		add_number(&script, 1);
 		add_op(&script, OP_CHECKSEQUENCEVERIFY);
 		add_op(&script, OP_DROP);
@@ -851,8 +768,7 @@ u8 *bitcoin_wscript_htlc_receive(const tal_t *ctx,
 				 const struct pubkey *remotehtlckey,
 				 const struct sha256 *payment_hash,
 				 const struct pubkey *revocationkey,
-				 bool option_anchor_outputs,
-				 bool option_anchors_zero_fee_htlc_tx)
+				 bool option_anchor_outputs)
 {
 	struct ripemd160 ripemd;
 
@@ -860,8 +776,7 @@ u8 *bitcoin_wscript_htlc_receive(const tal_t *ctx,
 	return bitcoin_wscript_htlc_receive_ripemd(ctx, htlc_abstimeout,
 						   localhtlckey, remotehtlckey,
 						   &ripemd, revocationkey,
-						   option_anchor_outputs,
-						   option_anchors_zero_fee_htlc_tx);
+						   option_anchor_outputs);
 }
 
 /* BOLT #3:
@@ -959,35 +874,15 @@ u8 *bitcoin_wscript_anchor(const tal_t *ctx,
 	add_op(&script, OP_CHECKSEQUENCEVERIFY);
 	add_op(&script, OP_ENDIF);
 
-	assert(is_anchor_witness_script(script, tal_bytelen(script)));
 	return script;
 }
 
-bool is_anchor_witness_script(const u8 *script, size_t script_len)
-{
-	if (script_len != 34 + 1 + 1 + 1 + 1 + 1 + 1)
-		return false;
-	if (script[0] != OP_PUSHBYTES(33))
-		return false;
-	if (script[34] != OP_CHECKSIG)
-		return false;
-	if (script[35] != OP_IFDUP)
-		return false;
-	if (script[36] != OP_NOTIF)
-		return false;
-	if (script[37] != 0x50 + 16)
-		return false;
-	if (script[38] != OP_CHECKSEQUENCEVERIFY)
-		return false;
-	if (script[39] != OP_ENDIF)
-		return false;
-	return true;
-}
-
 bool scripteq(const u8 *s1, const u8 *s2)
 {
-	size_t s1_len = tal_count(s1), s2_len = tal_count(s2);
-	memcheck(s1, s1_len);
-	memcheck(s2, s2_len);
-	return memeq(s1, s1_len, s2, s2_len);
+	memcheck(s1, tal_count(s1));
+	memcheck(s2, tal_count(s2));
+
+	if (tal_count(s1) != tal_count(s2))
+		return false;
+	return memcmp(s1, s2, tal_count(s1)) == 0;
 }

bitcoin/script.h

@@ -63,23 +63,10 @@ u8 *scriptpubkey_p2wpkh_derkey(const tal_t *ctx, const u8 der[33]);
 u8 *scriptpubkey_witness_raw(const tal_t *ctx, u8 version,
 			     const u8 *wprog, size_t wprog_size);
 
-/* Create an output script for a "raw"(perhaps already tweaked) taproot output pubkey */
-u8 *scriptpubkey_raw_p2tr(const tal_t *ctx, const struct pubkey *output_pubkey);
-
-/* Same as above, but compressed key is DER-encoded. */
-u8 *scriptpubkey_raw_p2tr_derkey(const tal_t *ctx, const u8 output_der[33]);
-
-/* Create an output script for an internal taproot pubkey. Results in different script than
- * scriptpubkey_raw_p2tr! TODO support merkle root tweaking */
-u8 *scriptpubkey_p2tr(const tal_t *ctx, const struct pubkey *inner_pubkey);
-
-/* Same as above, but compressed key is DER-encoded. TODO support merkle root tweaking */
-u8 *scriptpubkey_p2tr_derkey(const tal_t *ctx, const u8 inner_der[33]);
-
 /* To-remotekey with csv max(lease_expiry - blockheight, 1) delay. */
-u8 *bitcoin_wscript_to_remote_anchored(const tal_t *ctx,
-				       const struct pubkey *remote_key,
-				       u32 csv_lock);
+u8 *anchor_to_remote_redeem(const tal_t *ctx,
+			    const struct pubkey *remote_key,
+			    u32 csv_lock);
 
 /* Create a witness which spends the 2of2. */
 u8 **bitcoin_witness_2of2(const tal_t *ctx,
@@ -111,8 +98,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
 			       const struct pubkey *remotehtlckey,
 			       const struct sha256 *payment_hash,
 			       const struct pubkey *revocationkey,
-			       bool option_anchor_outputs,
-			       bool option_anchors_zero_fee_htlc_tx);
+			       bool option_anchor_outputs);
 u8 **bitcoin_witness_htlc_timeout_tx(const tal_t *ctx,
 				     const struct bitcoin_signature *localsig,
 				     const struct bitcoin_signature *remotesig,
@@ -123,8 +109,7 @@ u8 *bitcoin_wscript_htlc_receive(const tal_t *ctx,
 				 const struct pubkey *remotekey,
 				 const struct sha256 *payment_hash,
 				 const struct pubkey *revocationkey,
-				 bool option_anchor_outputs,
-				 bool option_anchors_zero_fee_htlc_tx);
+				 bool option_anchor_outputs);
 u8 **bitcoin_witness_htlc_success_tx(const tal_t *ctx,
 				     const struct bitcoin_signature *localsig,
 				     const struct bitcoin_signature *remotesig,
@@ -137,16 +122,14 @@ u8 *bitcoin_wscript_htlc_offer_ripemd160(const tal_t *ctx,
 					 const struct pubkey *remotehtlckey,
 					 const struct ripemd160 *payment_ripemd,
 					 const struct pubkey *revocationkey,
-					 bool option_anchor_outputs,
-					 bool option_anchors_zero_fee_htlc_tx);
+					 bool option_anchor_outputs);
 u8 *bitcoin_wscript_htlc_receive_ripemd(const tal_t *ctx,
 					const struct abs_locktime *htlc_abstimeout,
 					const struct pubkey *localkey,
 					const struct pubkey *remotekey,
 					const struct ripemd160 *payment_ripemd,
 					const struct pubkey *revocationkey,
-					bool option_anchor_outputs,
-					bool option_anchors_zero_fee_htlc_tx);
+					bool option_anchor_outputs);
 
 /* BOLT #3 HTLC-success/HTLC-timeout output */
 u8 *bitcoin_wscript_htlc_tx(const tal_t *ctx,
@@ -159,28 +142,19 @@ u8 *bitcoin_wscript_anchor(const tal_t *ctx,
 			   const struct pubkey *funding_pubkey);
 
 /* Is this a pay to pubkey hash? (extract addr if not NULL) */
-bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr);
+bool is_p2pkh(const u8 *script, struct bitcoin_address *addr);
 
 /* Is this a pay to script hash? (extract addr if not NULL) */
-bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr);
+bool is_p2sh(const u8 *script, struct ripemd160 *addr);
 
 /* Is this (version 0) pay to witness script hash? (extract addr if not NULL) */
-bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr);
+bool is_p2wsh(const u8 *script, struct sha256 *addr);
 
 /* Is this (version 0) pay to witness pubkey hash? (extract addr if not NULL) */
-bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr);
+bool is_p2wpkh(const u8 *script, struct bitcoin_address *addr);
 
-/* Is this a taproot output? (extract xonly_pubkey bytes if not NULL) */
-bool is_p2tr(const u8 *script, size_t script_len, u8 xonly_pubkey[32]);
-
-/* Is this one of the above script types? */
-bool is_known_scripttype(const u8 *script, size_t script_len);
-
-/* Is this a witness script type? */
-bool is_known_segwit_scripttype(const u8 *script, size_t script_len);
-
-/* Is this a to-remote witness script (used for option_anchor_outputs)? */
-bool is_to_remote_anchored_witness_script(const u8 *script, size_t script_len);
+/* Is this one of the four above script types? */
+bool is_known_scripttype(const u8 *script);
 
 /* Is this an anchor witness script? */
 bool is_anchor_witness_script(const u8 *script, size_t script_len);
@@ -203,7 +177,4 @@ void script_push_bytes(u8 **scriptp, const void *mem, size_t len);
 /* OP_0 + PUSH(32-byte-hash) */
 #define BITCOIN_SCRIPTPUBKEY_P2WSH_LEN (1 + 1 + 32)
 
-/* OP_1 + PUSH(32-byte-key) */
-#define BITCOIN_SCRIPTPUBKEY_P2TR_LEN (1 + 1 + 32)
-
 #endif /* LIGHTNING_BITCOIN_SCRIPT_H */

bitcoin/shadouble.c

@@ -1,7 +1,7 @@
 #include "config.h"
 #include <bitcoin/shadouble.h>
 #include <ccan/mem/mem.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
 void sha256_double(struct sha256_double *shadouble, const void *p, size_t len)
@@ -15,11 +15,7 @@ void sha256_double_done(struct sha256_ctx *shactx, struct sha256_double *res)
 	sha256_done(shactx, &res->sha);
 	sha256(&res->sha, &res->sha, sizeof(res->sha));
 }
-
-char *fmt_sha256_double(const tal_t *ctx, const struct sha256_double *shad)
-{
-	return tal_hexstr(ctx, shad, sizeof(*shad));
-}
+REGISTER_TYPE_TO_HEXSTR(sha256_double);
 
 void towire_sha256_double(u8 **pptr, const struct sha256_double *sha256d)
 {

bitcoin/shadouble.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/crypto/sha256/sha256.h>
 #include <ccan/short_types/short_types.h>
-#include <ccan/tal/tal.h>
 
 /* To explicitly distinguish between single sha and bitcoin's standard double */
 struct sha256_double {
@@ -18,6 +17,4 @@ void sha256_double_done(struct sha256_ctx *sha256, struct sha256_double *res);
 void fromwire_sha256_double(const u8 **cursor, size_t *max,
 			    struct sha256_double *sha256d);
 void towire_sha256_double(u8 **pptr, const struct sha256_double *sha256d);
-
-char *fmt_sha256_double(const tal_t *ctx, const struct sha256_double *shad);
 #endif /* LIGHTNING_BITCOIN_SHADOUBLE_H */

bitcoin/short_channel_id.c

@@ -1,6 +1,7 @@
 #include "config.h"
 #include <bitcoin/short_channel_id.h>
 #include <ccan/tal/str/str.h>
+#include <common/type_to_string.h>
 #include <stdio.h>
 #include <wire/wire.h>
 
@@ -52,7 +53,7 @@ bool short_channel_id_from_str(const char *str, size_t strlen,
 		&& mk_short_channel_id(dst, blocknum, txnum, outnum);
 }
 
-char *fmt_short_channel_id(const tal_t *ctx, struct short_channel_id scid)
+char *short_channel_id_to_str(const tal_t *ctx, const struct short_channel_id *scid)
 {
 	return tal_fmt(ctx, "%dx%dx%d",
 		       short_channel_id_blocknum(scid),
@@ -77,25 +78,26 @@ bool short_channel_id_dir_from_str(const char *str, size_t strlen,
 	return true;
 }
 
-char *fmt_short_channel_id_dir(const tal_t *ctx,
-			       const struct short_channel_id_dir *scidd)
+static char *short_channel_id_dir_to_str(const tal_t *ctx,
+					 const struct short_channel_id_dir *scidd)
 {
-	char *str, *scidstr = fmt_short_channel_id(NULL, scidd->scid);
+	char *str, *scidstr = short_channel_id_to_str(NULL, &scidd->scid);
 	str = tal_fmt(ctx, "%s/%u", scidstr, scidd->dir);
 	tal_free(scidstr);
 	return str;
 }
 
+REGISTER_TYPE_TO_STRING(short_channel_id, short_channel_id_to_str);
+REGISTER_TYPE_TO_STRING(short_channel_id_dir, short_channel_id_dir_to_str);
+
 void towire_short_channel_id(u8 **pptr,
-			     struct short_channel_id short_channel_id)
+			     const struct short_channel_id *short_channel_id)
 {
-	towire_u64(pptr, short_channel_id.u64);
+	towire_u64(pptr, short_channel_id->u64);
 }
 
-struct short_channel_id fromwire_short_channel_id(const u8 **cursor, size_t *max)
+void fromwire_short_channel_id(const u8 **cursor, size_t *max,
+			       struct short_channel_id *short_channel_id)
 {
-	struct short_channel_id scid;
-
-	scid.u64 = fromwire_u64(cursor, max);
-	return scid;
+	short_channel_id->u64 = fromwire_u64(cursor, max);
 }

bitcoin/short_channel_id.h

@@ -3,6 +3,7 @@
 #include "config.h"
 #include <ccan/compiler/compiler.h>
 #include <ccan/short_types/short_types.h>
+#include <ccan/structeq/structeq.h>
 #include <ccan/tal/tal.h>
 #include <common/gossip_constants.h>
 
@@ -11,18 +12,8 @@
 struct short_channel_id {
 	u64 u64;
 };
-
-static inline bool short_channel_id_eq(struct short_channel_id a,
-				       struct short_channel_id b)
-{
-	return a.u64 == b.u64;
-}
-
-static inline size_t short_channel_id_hash(struct short_channel_id scid)
-{
-	/* scids cost money to generate, so simple hash works here */
-	return (scid.u64 >> 32) ^ (scid.u64 >> 16) ^ scid.u64;
-}
+/* Define short_channel_id_eq (no padding) */
+STRUCTEQ_DEF(short_channel_id, 0, u64);
 
 /* BOLT #7:
  *
@@ -41,38 +32,32 @@ struct short_channel_id_dir {
 	int dir;
 };
 
-static inline bool short_channel_id_dir_eq(const struct short_channel_id_dir *a,
-					   const struct short_channel_id_dir *b)
+static inline u32 short_channel_id_blocknum(const struct short_channel_id *scid)
 {
-	return short_channel_id_eq(a->scid, b->scid) && a->dir == b->dir;
+	return scid->u64 >> 40;
 }
 
-static inline u32 short_channel_id_blocknum(struct short_channel_id scid)
+static inline bool is_stub_scid(const struct short_channel_id *scid)
 {
-	return scid.u64 >> 40;
+	return scid ? scid->u64 >> 40 == 1 &&
+		((scid->u64 >> 16) & 0x00FFFFFF) == 1 &&
+		(scid->u64 & 0xFFFF) == 1 : false;
 }
 
-static inline bool is_stub_scid(struct short_channel_id scid)
+static inline u32 short_channel_id_txnum(const struct short_channel_id *scid)
 {
-	return scid.u64 >> 40 == 1 &&
-		((scid.u64 >> 16) & 0x00FFFFFF) == 1 &&
-		(scid.u64 & 0xFFFF) == 1;
+	return (scid->u64 >> 16) & 0x00FFFFFF;
 }
 
-static inline u32 short_channel_id_txnum(struct short_channel_id scid)
+static inline u16 short_channel_id_outnum(const struct short_channel_id *scid)
 {
-	return (scid.u64 >> 16) & 0x00FFFFFF;
-}
-
-static inline u16 short_channel_id_outnum(struct short_channel_id scid)
-{
-	return scid.u64 & 0xFFFF;
+	return scid->u64 & 0xFFFF;
 }
 
 /* Subtly, at block N, depth is 1, hence the -1 here. eg. 103x1x0 is announceable
  * when height is 108. */
 static inline bool
-is_scid_depth_announceable(struct short_channel_id scid,
+is_scid_depth_announceable(const struct short_channel_id *scid,
 			  unsigned int height)
 {
 	return short_channel_id_blocknum(scid) + ANNOUNCE_MIN_DEPTH - 1
@@ -86,16 +71,15 @@ bool WARN_UNUSED_RESULT mk_short_channel_id(struct short_channel_id *scid,
 bool WARN_UNUSED_RESULT short_channel_id_from_str(const char *str, size_t strlen,
 						  struct short_channel_id *dst);
 
+char *short_channel_id_to_str(const tal_t *ctx, const struct short_channel_id *scid);
+
 bool WARN_UNUSED_RESULT short_channel_id_dir_from_str(const char *str, size_t strlen,
 						      struct short_channel_id_dir *scidd);
 
-char *fmt_short_channel_id(const tal_t *ctx, struct short_channel_id scid);
-char *fmt_short_channel_id_dir(const tal_t *ctx,
-			       const struct short_channel_id_dir *scidd);
-
 /* Marshal/unmarshal */
 void towire_short_channel_id(u8 **pptr,
-			     struct short_channel_id short_channel_id);
-struct short_channel_id fromwire_short_channel_id(const u8 **cursor, size_t *max);
+			     const struct short_channel_id *short_channel_id);
+void fromwire_short_channel_id(const u8 **cursor, size_t *max,
+			       struct short_channel_id *short_channel_id);
 
 #endif /* LIGHTNING_BITCOIN_SHORT_CHANNEL_ID_H */

bitcoin/signature.c

@@ -8,7 +8,7 @@
 #include <bitcoin/signature.h>
 #include <bitcoin/tx.h>
 #include <ccan/mem/mem.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <secp256k1_schnorrsig.h>
 #include <wire/wire.h>
 
@@ -94,7 +94,12 @@ static bool sig_has_low_r(const secp256k1_ecdsa_signature* sig)
 	return compact_sig[0] < 0x80;
 }
 
-bool dev_no_signature_grind = false;
+#if DEVELOPER
+/* Some of the spec test vectors assume no sig grinding. */
+extern bool dev_no_grind;
+
+bool dev_no_grind = false;
+#endif
 
 void sign_hash(const struct privkey *privkey,
 	       const struct sha256_double *h,
@@ -109,10 +114,11 @@ void sign_hash(const struct privkey *privkey,
 					  s,
 					  h->sha.u.u8,
 					  privkey->secret.data, NULL,
-					  dev_no_signature_grind ? NULL
-					  : extra_entropy);
+					  IFDEV(dev_no_grind ? NULL
+						: extra_entropy,
+						extra_entropy));
 		((u32 *)extra_entropy)[0]++;
-		if (dev_no_signature_grind)
+		if (IFDEV(dev_no_grind, false))
 			break;
 	} while (!sig_has_low_r(s));
 
@@ -324,7 +330,7 @@ bool signature_from_der(const u8 *der, size_t len, struct bitcoin_signature *sig
 	return true;
 }
 
-char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx, const secp256k1_ecdsa_signature *sig)
+char *fmt_signature(const tal_t *ctx, const secp256k1_ecdsa_signature *sig)
 {
 	u8 der[72];
 	size_t len = 72;
@@ -334,15 +340,17 @@ char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx, const secp256k1_ecdsa_sign
 
 	return tal_hexstr(ctx, der, len);
 }
+REGISTER_TYPE_TO_STRING(secp256k1_ecdsa_signature, fmt_signature);
 
-char *fmt_bitcoin_signature(const tal_t *ctx,
-			    const struct bitcoin_signature *sig)
+static char *bitcoin_signature_to_hexstr(const tal_t *ctx,
+					 const struct bitcoin_signature *sig)
 {
 	u8 der[73];
 	size_t len = signature_to_der(der, sig);
 
 	return tal_hexstr(ctx, der, len);
 }
+REGISTER_TYPE_TO_STRING(bitcoin_signature, bitcoin_signature_to_hexstr);
 
 void fromwire_bitcoin_signature(const u8 **cursor, size_t *max,
 				struct bitcoin_signature *sig)
@@ -376,6 +384,8 @@ char *fmt_bip340sig(const tal_t *ctx, const struct bip340sig *bip340sig)
 	return tal_hexstr(ctx, bip340sig->u8, sizeof(bip340sig->u8));
 }
 
+REGISTER_TYPE_TO_HEXSTR(bip340sig);
+
 /* BIP-340:
  *
  * This proposal suggests to include the tag by prefixing the hashed

bitcoin/signature.h

@@ -150,11 +150,8 @@ void fromwire_bip340sig(const u8 **cursor, size_t *max,
 			struct bip340sig *bip340sig);
 
 /* Get a hex string sig */
-char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx,
-				    const secp256k1_ecdsa_signature *sig);
+char *fmt_signature(const tal_t *ctx, const secp256k1_ecdsa_signature *sig);
 char *fmt_bip340sig(const tal_t *ctx, const struct bip340sig *bip340sig);
-char *fmt_bitcoin_signature(const tal_t *ctx,
-			    const struct bitcoin_signature *sig);
 
 /* For caller convenience, we hand in tag in parts (any can be "") */
 void bip340_sighash_init(struct sha256_ctx *sctx,
@@ -162,7 +159,4 @@ void bip340_sighash_init(struct sha256_ctx *sctx,
 			 const char *tag2,
 			 const char *tag3);
 
-/* Some of the spec test vectors assume no sig grinding. */
-extern bool dev_no_signature_grind;
-
 #endif /* LIGHTNING_BITCOIN_SIGNATURE_H */

bitcoin/test/run-bitcoin_block_from_hex.c

@@ -16,9 +16,6 @@ bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
 /* Generated stub for amount_sat */
 struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 { fprintf(stderr, "amount_sat called!\n"); abort(); }
@@ -69,15 +66,15 @@ void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr
 /* Generated stub for is_anchor_witness_script */
 bool is_anchor_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
 { fprintf(stderr, "is_anchor_witness_script called!\n"); abort(); }
-/* Generated stub for is_to_remote_anchored_witness_script */
-bool is_to_remote_anchored_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_to_remote_anchored_witness_script called!\n"); abort(); }
 /* Generated stub for pubkey_to_der */
 void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UNNEEDED)
 { fprintf(stderr, "pubkey_to_der called!\n"); abort(); }
 /* Generated stub for pubkey_to_hash160 */
 void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
 { fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
+/* Generated stub for script_push_bytes */
+void script_push_bytes(u8 **scriptp UNNEEDED, const void *mem UNNEEDED, size_t len UNNEEDED)
+{ fprintf(stderr, "script_push_bytes called!\n"); abort(); }
 /* Generated stub for scriptpubkey_p2wsh */
 u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
 { fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }
@@ -171,13 +168,13 @@ int main(int argc, const char *argv[])
 				   block, strlen(block));
 
 	assert(b);
-	assert(b->hdr.version == 0x6592a000);
+	assert(b->hdr.version == CPU_TO_LE32(0x6592a000));
 	bitcoin_blkid_from_hex("0000000000000f31173e973bc00e452b1fac350066df7db2adec1e3224ea5bc1", strlen("0000000000000f31173e973bc00e452b1fac350066df7db2adec1e3224ea5bc1"), &prev);
 	assert(bitcoin_blkid_eq(&prev, &b->hdr.prev_hash));
 	hex_decode("8a0ee58ded5de949325ebc99583e3ca84f96a6597465c611685413f50f0ead7e", strlen("8a0ee58ded5de949325ebc99583e3ca84f96a6597465c611685413f50f0ead7e"), &merkle, sizeof(merkle));
 	assert(sha256_double_eq(&merkle, &b->hdr.merkle_hash));
-	assert(b->hdr.timestamp == 1550507183);
-	assert(b->hdr.nonce == 1226407989);
+	assert(b->hdr.timestamp == CPU_TO_LE32(1550507183));
+	assert(b->hdr.nonce == CPU_TO_LE32(1226407989));
 
 	assert(tal_count(b->tx) == 3);
 	bitcoin_txid(b->tx[0], &txid);

bitcoin/test/run-psbt-from-tx.c

@@ -1,121 +0,0 @@
-#include "config.h"
-#include "../psbt.c"
-#include "../tx.c"
-#include "../../wire/towire.c"
-#include "../../wire/fromwire.c"
-#include <assert.h>
-#include <ccan/cast/cast.h>
-#include <common/setup.h>
-#include <stdio.h>
-
-/* AUTOGENERATED MOCKS START */
-/* Generated stub for amount_asset_is_main */
-bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
-{ fprintf(stderr, "amount_asset_is_main called!\n"); abort(); }
-/* Generated stub for amount_asset_to_sat */
-struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
-{ fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
-/* Generated stub for amount_sat */
-struct amount_sat amount_sat(u64 satoshis UNNEEDED)
-{ fprintf(stderr, "amount_sat called!\n"); abort(); }
-/* Generated stub for amount_sat_add */
- bool amount_sat_add(struct amount_sat *val UNNEEDED,
-				       struct amount_sat a UNNEEDED,
-				       struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_add called!\n"); abort(); }
-/* Generated stub for amount_sat_eq */
-bool amount_sat_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_eq called!\n"); abort(); }
-/* Generated stub for amount_sat_greater_eq */
-bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); }
-/* Generated stub for amount_sat_sub */
- bool amount_sat_sub(struct amount_sat *val UNNEEDED,
-				       struct amount_sat a UNNEEDED,
-				       struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_sub called!\n"); abort(); }
-/* Generated stub for amount_sat_to_asset */
-struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u8 *asset UNNEEDED)
-{ fprintf(stderr, "amount_sat_to_asset called!\n"); abort(); }
-/* Generated stub for amount_tx_fee */
-struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_tx_fee called!\n"); abort(); }
-/* Generated stub for fromwire_sha256_double */
-void fromwire_sha256_double(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-			    struct sha256_double *sha256d UNNEEDED)
-{ fprintf(stderr, "fromwire_sha256_double called!\n"); abort(); }
-/* Generated stub for is_anchor_witness_script */
-bool is_anchor_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_anchor_witness_script called!\n"); abort(); }
-/* Generated stub for is_to_remote_anchored_witness_script */
-bool is_to_remote_anchored_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_to_remote_anchored_witness_script called!\n"); abort(); }
-/* Generated stub for pubkey_to_der */
-void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UNNEEDED)
-{ fprintf(stderr, "pubkey_to_der called!\n"); abort(); }
-/* Generated stub for pubkey_to_hash160 */
-void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
-{ fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
-/* Generated stub for scriptpubkey_p2wsh */
-u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
-{ fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }
-/* Generated stub for sha256_double */
-void sha256_double(struct sha256_double *shadouble UNNEEDED, const void *p UNNEEDED, size_t len UNNEEDED)
-{ fprintf(stderr, "sha256_double called!\n"); abort(); }
-/* Generated stub for signature_from_der */
-bool signature_from_der(const u8 *der UNNEEDED, size_t len UNNEEDED, struct bitcoin_signature *sig UNNEEDED)
-{ fprintf(stderr, "signature_from_der called!\n"); abort(); }
-/* Generated stub for signature_to_der */
-size_t signature_to_der(u8 der[73] UNNEEDED, const struct bitcoin_signature *sig UNNEEDED)
-{ fprintf(stderr, "signature_to_der called!\n"); abort(); }
-/* Generated stub for towire_sha256_double */
-void towire_sha256_double(u8 **pptr UNNEEDED, const struct sha256_double *sha256d UNNEEDED)
-{ fprintf(stderr, "towire_sha256_double called!\n"); abort(); }
-/* Generated stub for varint_put */
-size_t varint_put(u8 buf[VARINT_MAX_LEN] UNNEEDED, varint_t v UNNEEDED)
-{ fprintf(stderr, "varint_put called!\n"); abort(); }
-/* Generated stub for varint_size */
-size_t varint_size(varint_t v UNNEEDED)
-{ fprintf(stderr, "varint_size called!\n"); abort(); }
-/* AUTOGENERATED MOCKS END */
-
-/* This transaction has scriptSig data in it.
- * We expect that creating a new psbt from it will correctly
- * populate the PSBT object */
-static const char *raw_tx = "0200000000010151d12aa54cc6e59a6a92325a8315e93361d9805115a13aa5ba8dbcf30ffd858c000000001716001401fad90abcd66697e2592164722de4a95ebee165fdffffff02603c250200000000160014c2ccab171c2a5be9dab52ec41b825863024c546600093d00000000002200205b8cd3b914cf67cdd8fa6273c930353dd36476734fbd962102c2df53b90880cd02473044022001e73b1745d775521c758e70549ad79b1d076efc34303f416e66ff630f6088e402207b0aa44b35329ae4733463bc9f6ca433c5595f00a902a21c941945a24f8aa577012103d745445c9362665f22e0d96e9e766f273f3260dea39c8a76bfa05dd2684ddccf66000000";
-
-int main(int argc, char *argv[])
-{
-	struct bitcoin_tx *tx, *tx2;
-	u8 *msg;
-	size_t len;
-
-	common_setup(argv[0]);
-	chainparams = chainparams_for_network("bitcoin");
-
-	msg = tal_arr(tmpctx, u8, 0);
-	tx = bitcoin_tx_from_hex(tmpctx, raw_tx, strlen(raw_tx));
-
-	/* convert to wire format */
-	towire_bitcoin_tx(&msg, tx);
-
-	len = tal_bytelen(msg);
-	assert(len > 0);
-
-	tx2 = fromwire_bitcoin_tx(tmpctx,
-				  cast_const2(const u8 **, &msg), &len);
-	assert(tx2 != NULL);
-
-	/* Witness/scriptsig data is saved down into psbt */
-	assert(tx2->psbt->num_inputs == 1);
-	const struct wally_map_item *final_scriptsig = wally_map_get_integer(&tx2->psbt->inputs[0].psbt_fields, /* PSBT_IN_FINAL_SCRIPTSIG */ 0x07);
-	assert(final_scriptsig->value_len > 0);
-	assert(tx2->psbt->inputs[0].final_witness != NULL);
-	audit_psbt(tx2->psbt, tx2->psbt);
-
-	common_shutdown();
-	return 0;
-}

bitcoin/test/run-tx-bitcoin_tx_2of2_input_witness_weight.c

@@ -19,9 +19,6 @@ bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
 /* Generated stub for amount_sat_add */
  bool amount_sat_add(struct amount_sat *val UNNEEDED,
 				       struct amount_sat a UNNEEDED,
@@ -41,9 +38,6 @@ struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u
 /* Generated stub for amount_tx_fee */
 struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED)
 { fprintf(stderr, "amount_tx_fee called!\n"); abort(); }
-/* Generated stub for clone_psbt */
-struct wally_psbt *clone_psbt(const tal_t *ctx UNNEEDED, const struct wally_psbt *psbt UNNEEDED)
-{ fprintf(stderr, "clone_psbt called!\n"); abort(); }
 /* Generated stub for fromwire */
 const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED)
 { fprintf(stderr, "fromwire called!\n"); abort(); }
@@ -87,6 +81,10 @@ struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt UNNEEDED,
 					   const u8 *input_wscript UNNEEDED,
 					   const u8 *redeemscript UNNEEDED)
 { fprintf(stderr, "psbt_append_input called!\n"); abort(); }
+/* Generated stub for psbt_elements_input_set_asset */
+void psbt_elements_input_set_asset(struct wally_psbt *psbt UNNEEDED, size_t in UNNEEDED,
+				   struct amount_asset *asset UNNEEDED)
+{ fprintf(stderr, "psbt_elements_input_set_asset called!\n"); abort(); }
 /* Generated stub for psbt_final_tx */
 struct wally_tx *psbt_final_tx(const tal_t *ctx UNNEEDED, const struct wally_psbt *psbt UNNEEDED)
 { fprintf(stderr, "psbt_final_tx called!\n"); abort(); }
@@ -147,14 +145,6 @@ int main(int argc, const char *argv[])
 
 	/* 1 byte for num witnesses, one per witness element */
 	weight = 1;
-
-	/* Two signatures, slightly overestimated to be 73 bytes each,
-	 * while the actual witness will often be smaller.*/
-        /* BOLT #03:
-         * Signatures are 73 bytes long (the maximum length).
-         */
-	weight += 2 + 2;
-
 	for (size_t i = 0; i < tal_count(wit); i++)
 		weight += 1 + tal_bytelen(wit[i]);
 	assert(bitcoin_tx_2of2_input_witness_weight() == weight);

bitcoin/test/run-tx-encode.c

@@ -17,9 +17,6 @@ bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
 /* Generated stub for amount_sat */
 struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 { fprintf(stderr, "amount_sat called!\n"); abort(); }
@@ -70,15 +67,15 @@ void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr
 /* Generated stub for is_anchor_witness_script */
 bool is_anchor_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
 { fprintf(stderr, "is_anchor_witness_script called!\n"); abort(); }
-/* Generated stub for is_to_remote_anchored_witness_script */
-bool is_to_remote_anchored_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_to_remote_anchored_witness_script called!\n"); abort(); }
 /* Generated stub for pubkey_to_der */
 void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UNNEEDED)
 { fprintf(stderr, "pubkey_to_der called!\n"); abort(); }
 /* Generated stub for pubkey_to_hash160 */
 void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
 { fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
+/* Generated stub for script_push_bytes */
+void script_push_bytes(u8 **scriptp UNNEEDED, const void *mem UNNEEDED, size_t len UNNEEDED)
+{ fprintf(stderr, "script_push_bytes called!\n"); abort(); }
 /* Generated stub for scriptpubkey_p2wsh */
 u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
 { fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }

bitcoin/tx.c

@@ -3,10 +3,9 @@
 #include <bitcoin/psbt.h>
 #include <bitcoin/script.h>
 #include <bitcoin/tx.h>
-#include <ccan/cast/cast.h>
 #include <ccan/str/hex/hex.h>
 #include <ccan/tal/str/str.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wally_psbt.h>
 #include <wire/wire.h>
 
@@ -95,15 +94,6 @@ int bitcoin_tx_add_output(struct bitcoin_tx *tx, const u8 *script,
 	return i;
 }
 
-void bitcoin_tx_remove_output(struct bitcoin_tx *tx, size_t outnum)
-{
-	int ret;
-	ret = wally_tx_remove_output(tx->wtx, outnum);
-	assert(ret == WALLY_OK);
-	ret = wally_psbt_remove_output(tx->psbt, outnum);
-	assert(ret == WALLY_OK);
-}
-
 bool elements_wtx_output_is_fee(const struct wally_tx *tx, int outnum)
 {
 	assert(outnum < tx->num_outputs);
@@ -190,36 +180,7 @@ static int elements_tx_add_fee_output(struct bitcoin_tx *tx)
 void bitcoin_tx_set_locktime(struct bitcoin_tx *tx, u32 locktime)
 {
 	tx->wtx->locktime = locktime;
-	tx->psbt->fallback_locktime = locktime;
-	tx->psbt->has_fallback_locktime = true;
-}
-
-/* FIXME Stolen from psbt_append_input; export? */
-static struct wally_tx_input *wally_tx_input_from_outpoint_sequence(const struct bitcoin_outpoint *outpoint,
-			u32 sequence)
-{
-	struct wally_tx_input *tx_in;
-    if (chainparams->is_elements) {
-        if (wally_tx_elements_input_init_alloc(outpoint->txid.shad.sha.u.u8,
-                               sizeof(outpoint->txid.shad.sha.u.u8),
-                               outpoint->n,
-                               sequence, NULL, 0,
-                               NULL,
-                               NULL, 0,
-                               NULL, 0, NULL, 0,
-                               NULL, 0, NULL, 0,
-                               NULL, 0, NULL,
-                               &tx_in) != WALLY_OK)
-            abort();
-    } else {
-        if (wally_tx_input_init_alloc(outpoint->txid.shad.sha.u.u8,
-                          sizeof(outpoint->txid.shad.sha.u.u8),
-                          outpoint->n,
-                          sequence, NULL, 0, NULL,
-                          &tx_in) != WALLY_OK)
-            abort();
-    }
-	return tx_in;
+	tx->psbt->tx->locktime = locktime;
 }
 
 int bitcoin_tx_add_input(struct bitcoin_tx *tx,
@@ -230,7 +191,6 @@ int bitcoin_tx_add_input(struct bitcoin_tx *tx,
 {
 	int wally_err;
 	int input_num = tx->wtx->num_inputs;
-	struct wally_tx_input *tx_input;
 
 	psbt_append_input(tx->psbt, outpoint,
 			  sequence, scriptSig,
@@ -245,11 +205,9 @@ int bitcoin_tx_add_input(struct bitcoin_tx *tx,
 				scriptPubkey, amount);
 
 	tal_wally_start();
-	tx_input = wally_tx_input_from_outpoint_sequence(outpoint, sequence);
 	wally_err = wally_tx_add_input(tx->wtx,
-			       tx_input);
+				       &tx->psbt->tx->inputs[input_num]);
 	assert(wally_err == WALLY_OK);
-	wally_tx_input_free(tx_input);
 
 	/* scriptsig isn't actually stored in psbt input, so add that now */
 	wally_tx_set_input_script(tx->wtx, input_num,
@@ -257,10 +215,12 @@ int bitcoin_tx_add_input(struct bitcoin_tx *tx,
 	tal_wally_end(tx->wtx);
 
 	if (is_elements(chainparams)) {
+		struct amount_asset asset;
 		/* FIXME: persist asset tags */
-		amount_sat_to_asset(&amount,
+		asset = amount_sat_to_asset(&amount,
 					    chainparams->fee_asset_tag);
 		/* FIXME: persist nonces */
+		psbt_elements_input_set_asset(tx->psbt, input_num, &asset);
 	}
 	return input_num;
 }
@@ -298,24 +258,47 @@ void bitcoin_tx_output_set_amount(struct bitcoin_tx *tx, int outnum,
 		assert(ret == WALLY_OK);
 	} else {
 		output->satoshi = satoshis;
+
+		/* update the global tx for the psbt also */
+		output = &tx->psbt->tx->outputs[outnum];
+		output->satoshi = satoshis;
 	}
-	wally_psbt_output_set_amount(&tx->psbt->outputs[outnum], satoshis);
+}
+
+const u8 *wally_tx_output_get_script(const tal_t *ctx,
+				     const struct wally_tx_output *output)
+{
+	if (output->script == NULL) {
+		/* This can happen for coinbase transactions and pegin
+		 * transactions */
+		return NULL;
+	}
+
+	return tal_dup_arr(ctx, u8, output->script, output->script_len, 0);
+}
+
+const u8 *bitcoin_tx_output_get_script(const tal_t *ctx,
+				       const struct bitcoin_tx *tx, int outnum)
+{
+	const struct wally_tx_output *output;
+	assert(outnum < tx->wtx->num_outputs);
+	output = &tx->wtx->outputs[outnum];
+
+	return wally_tx_output_get_script(ctx, output);
 }
 
 u8 *bitcoin_tx_output_get_witscript(const tal_t *ctx, const struct bitcoin_tx *tx,
 				    int outnum)
 {
 	struct wally_psbt_output *out;
-	const struct wally_map_item *output_witness_script;
 
 	assert(outnum < tx->psbt->num_outputs);
 	out = &tx->psbt->outputs[outnum];
 
-	output_witness_script = wally_map_get_integer(&out->psbt_fields, /* PSBT_OUT_WITNESS_SCRIPT */ 0x01);
-	if (output_witness_script->value_len == 0)
+	if (out->witness_script_len == 0)
 		return NULL;
 
-	return tal_dup_arr(ctx, u8, output_witness_script->value, output_witness_script->value_len, 0);
+	return tal_dup_arr(ctx, u8, out->witness_script, out->witness_script_len, 0);
 }
 
 struct amount_asset bitcoin_tx_output_get_amount(const struct bitcoin_tx *tx,
@@ -403,7 +386,8 @@ void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
 	assert(innum < tx->wtx->num_inputs);
 
 	in = &tx->wtx->inputs[innum];
-	CROSS_TYPE_ASSIGNMENT(&in->txhash, &outpoint->txid);
+	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+	memcpy(in->txhash, &outpoint->txid, sizeof(struct bitcoin_txid));
 	in->index = outpoint->n;
 }
 
@@ -411,13 +395,15 @@ void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
 void wally_tx_input_get_txid(const struct wally_tx_input *in,
 			     struct bitcoin_txid *txid)
 {
-	CROSS_TYPE_ASSIGNMENT(txid, &in->txhash);
+	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+	memcpy(txid, in->txhash, sizeof(struct bitcoin_txid));
 }
 
 void wally_tx_input_get_outpoint(const struct wally_tx_input *in,
 				 struct bitcoin_outpoint *outpoint)
 {
-	wally_tx_input_get_txid(in, &outpoint->txid);
+	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+	memcpy(&outpoint->txid, in->txhash, sizeof(struct bitcoin_txid));
 	outpoint->n = in->index;
 }
 
@@ -548,27 +534,20 @@ void bitcoin_tx_finalize(struct bitcoin_tx *tx)
 	assert(bitcoin_tx_check(tx));
 }
 
-struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt TAKES)
+struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt STEALS)
 {
-	size_t locktime;
-
-	if (!taken(psbt))
-		psbt = clone_psbt(tmpctx, psbt);
-
-	wally_psbt_get_locktime(psbt, &locktime);
 	struct bitcoin_tx *tx = bitcoin_tx(ctx, chainparams,
-					   psbt->num_inputs,
-					   psbt->num_outputs,
-					   locktime);
+					   psbt->tx->num_inputs,
+					   psbt->tx->num_outputs,
+					   psbt->tx->locktime);
 	wally_tx_free(tx->wtx);
 
 	psbt_finalize(psbt);
 	tx->wtx = psbt_final_tx(tx, psbt);
 	if (!tx->wtx) {
 		tal_wally_start();
-		if (wally_psbt_extract(psbt, WALLY_PSBT_EXTRACT_NON_FINAL, &tx->wtx) != WALLY_OK) {
+		if (wally_tx_clone_alloc(psbt->tx, 0, &tx->wtx) != WALLY_OK)
 			tx->wtx = NULL;
-		}
 		tal_wally_end_onto(tx, tx->wtx, struct wally_tx);
 		if (!tx->wtx)
 			return tal_free(tx);
@@ -580,30 +559,6 @@ struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psb
 	return tx;
 }
 
-struct bitcoin_tx *clone_bitcoin_tx(const tal_t *ctx,
-				    const struct bitcoin_tx *tx)
-{
-	struct bitcoin_tx *newtx;
-
-	if (taken(tx))
-		return cast_const(struct bitcoin_tx *, tal_steal(ctx, tx));
-
-	newtx = tal(ctx, struct bitcoin_tx);
-
-	newtx->chainparams = tx->chainparams;
-
-	tal_wally_start();
-	if (wally_tx_clone_alloc(tx->wtx, 0, &newtx->wtx) != WALLY_OK)
-		newtx->wtx = NULL;
-	tal_wally_end_onto(newtx, newtx->wtx, struct wally_tx);
-	if (!newtx->wtx)
-		return tal_free(newtx);
-
-	newtx->psbt = clone_psbt(newtx, tx->psbt);
-	tal_add_destructor(newtx, bitcoin_tx_destroy);
-	return newtx;
-}
-
 static struct wally_tx *pull_wtx(const tal_t *ctx,
 				 const u8 **cursor,
 				 size_t *max)
@@ -631,8 +586,8 @@ static struct wally_tx *pull_wtx(const tal_t *ctx,
 	return wtx;
 }
 
-struct bitcoin_tx *pull_bitcoin_tx_only(const tal_t *ctx, const u8 **cursor,
-					size_t *max)
+struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx, const u8 **cursor,
+				   size_t *max)
 {
 	struct bitcoin_tx *tx = tal(ctx, struct bitcoin_tx);
 	tx->wtx = pull_wtx(tx, cursor, max);
@@ -641,23 +596,13 @@ struct bitcoin_tx *pull_bitcoin_tx_only(const tal_t *ctx, const u8 **cursor,
 
 	tal_add_destructor(tx, bitcoin_tx_destroy);
 	tx->chainparams = chainparams;
-	tx->psbt = NULL;
+	tx->psbt = new_psbt(tx, tx->wtx);
+	if (!tx->psbt)
+		return tal_free(tx);
 
 	return tx;
 }
 
-struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx, const u8 **cursor,
-				   size_t *max)
-{
-	struct bitcoin_tx *tx = pull_bitcoin_tx_only(ctx, cursor, max);
-	if (tx) {
-		tx->psbt = new_psbt(tx, tx->wtx);
-		if (!tx->psbt)
-			return tal_free(tx);
-        }
-	return tx;
-}
-
 struct bitcoin_tx *bitcoin_tx_from_hex(const tal_t *ctx, const char *hex,
 				       size_t hexlen)
 {
@@ -694,6 +639,18 @@ fail:
 	return NULL;
 }
 
+/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
+static void reverse_bytes(u8 *arr, size_t len)
+{
+	unsigned int i;
+
+	for (i = 0; i < len / 2; i++) {
+		unsigned char tmp = arr[i];
+		arr[i] = arr[len - 1 - i];
+		arr[len - 1 - i] = tmp;
+	}
+}
+
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid)
 {
@@ -711,7 +668,7 @@ bool bitcoin_txid_to_hex(const struct bitcoin_txid *txid,
 	return hex_encode(&rev, sizeof(rev), hexstr, hexstr_len);
 }
 
-char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx)
+static char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx)
 {
 	u8 *lin = linearize_tx(ctx, tx);
 	char *s = tal_hex(ctx, lin);
@@ -719,7 +676,7 @@ char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx)
 	return s;
 }
 
-char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
+static char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
 {
 	char *hexstr = tal_arr(ctx, char, hex_str_size(sizeof(*txid)));
 
@@ -727,15 +684,15 @@ char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
 	return hexstr;
 }
 
-char *fmt_bitcoin_outpoint(const tal_t *ctx,
-			   const struct bitcoin_outpoint *outpoint)
+static char *fmt_bitcoin_outpoint(const tal_t *ctx,
+				  const struct bitcoin_outpoint *outpoint)
 {
 	return tal_fmt(ctx, "%s:%u",
 		       fmt_bitcoin_txid(tmpctx, &outpoint->txid),
 		       outpoint->n);
 }
 
-char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
+static char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
 {
 	u8 *lin = linearize_wtx(ctx, tx);
 	char *s = tal_hex(ctx, lin);
@@ -743,15 +700,10 @@ char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
 	return s;
 }
 
-char *fmt_sha256(const tal_t *ctx, const struct sha256 *sha256)
-{
-	return tal_hexstr(ctx, sha256, sizeof(*sha256));
-}
-
-char *fmt_ripemd160(const tal_t *ctx, const struct ripemd160 *ripemd160)
-{
-	return tal_hexstr(ctx, ripemd160, sizeof(*ripemd160));
-}
+REGISTER_TYPE_TO_STRING(bitcoin_tx, fmt_bitcoin_tx);
+REGISTER_TYPE_TO_STRING(bitcoin_txid, fmt_bitcoin_txid);
+REGISTER_TYPE_TO_STRING(bitcoin_outpoint, fmt_bitcoin_outpoint);
+REGISTER_TYPE_TO_STRING(wally_tx, fmt_wally_tx);
 
 void fromwire_bitcoin_txid(const u8 **cursor, size_t *max,
 			   struct bitcoin_txid *txid)
@@ -766,13 +718,15 @@ struct bitcoin_tx *fromwire_bitcoin_tx(const tal_t *ctx,
 
 	u32 len = fromwire_u32(cursor, max);
 	size_t start = *max;
-	tx = pull_bitcoin_tx_only(ctx, cursor, max);
+	tx = pull_bitcoin_tx(ctx, cursor, max);
 	if (!tx)
 		return fromwire_fail(cursor, max);
 	// Check that we consumed len bytes
 	if (start - *max != len)
 		return fromwire_fail(cursor, max);
 
+	/* pull_bitcoin_tx sets the psbt */
+	tal_free(tx->psbt);
 	tx->psbt = fromwire_wally_psbt(tx, cursor, max);
 	if (!tx->psbt)
 		return fromwire_fail(cursor, max);
@@ -813,7 +767,8 @@ bool wally_tx_input_spends(const struct wally_tx_input *input,
 	/* Useful, as tx_part can have some NULL inputs */
 	if (!input)
 		return false;
-	if (!CROSS_TYPE_EQ(&outpoint->txid, &input->txhash))
+	BUILD_ASSERT(sizeof(outpoint->txid) == sizeof(input->txhash));
+	if (memcmp(&outpoint->txid, input->txhash, sizeof(outpoint->txid)) != 0)
 		return false;
 	return input->index == outpoint->n;
 }
@@ -897,8 +852,7 @@ size_t bitcoin_tx_input_sig_weight(void)
 /* Input weight */
 size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight)
 {
-	/* We assume < 253 witness elements */
-	size_t weight = 1 + witness_weight;
+	size_t weight = witness_weight;
 
 	/* Input weight: txid + index + sequence */
 	weight += (32 + 4 + 4) * 4;
@@ -917,74 +871,45 @@ size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight)
 	return weight;
 }
 
-size_t bitcoin_tx_input_witness_weight(enum utxotype utxotype)
+size_t bitcoin_tx_simple_input_witness_weight(void)
 {
-	switch (utxotype) {
-	case UTXO_P2SH_P2WPKH:
-	case UTXO_P2WPKH:
-		/* Account for witness (sig + key) */
-		return bitcoin_tx_input_sig_weight() + 1 + 33;
-	case UTXO_P2WSH_FROM_CLOSE:
-		/* BOLT #3:
-		 * #### `to_remote` Output
-		 *
-		 * If `option_anchors` applies to the commitment
-		 * transaction, the `to_remote` output is encumbered by a one
-		 * block csv lock.
-		 *    <remotepubkey> OP_CHECKSIGVERIFY 1 OP_CHECKSEQUENCEVERIFY
-		 *
-		 * The output is spent by an input with `nSequence` field set
-		 * to `1` and witness: <remote_sig>
-		 * Otherwise, this output is a simple P2WPKH to `remotepubkey`.
-		 */
-		/* In practice, these predate anchors, so: */
-		return 1 + 1 + bitcoin_tx_input_sig_weight();
-	case UTXO_P2TR:
-		return 1 + 64;
-	}
-	abort();
+	/* Account for witness (1 byte count + sig + key) */
+	return 1 + (bitcoin_tx_input_sig_weight() + 1 + 33);
+}
+
+/* We only do segwit inputs, and we assume witness is sig + key  */
+size_t bitcoin_tx_simple_input_weight(bool p2sh)
+{
+	return bitcoin_tx_input_weight(p2sh,
+				       bitcoin_tx_simple_input_witness_weight());
 }
 
 size_t bitcoin_tx_2of2_input_witness_weight(void)
 {
-        /* BOLT #03:
-         * Signatures are 73 bytes long (the maximum length).
-         */
-	return 1 + /* Prefix: 4 elements to push on stack */
-		(1 + 0) + /* [0]: witness-marker-and-flag */
-		(1 + 73) + /* [1] Party A signature and length prefix */
-		(1 + 73) + /* [2] Party B signature and length prefix */
-		(1 + 1 + /* [3] length prefix and numpushes (2) */
-		 1 + 33 + /* pubkey A (with prefix) */
-		 1 + 33 + /* pubkey B (with prefix) */
-		 1 + 1 /* num sigs required and checkmultisig */
-		);
-}
-
-size_t change_weight(void)
-{
-	return bitcoin_tx_output_weight(chainparams->is_elements ? BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN : BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
-}
-
-struct amount_sat change_fee(u32 feerate_perkw,	size_t total_weight)
-{
-	struct amount_sat fee;
-
-	/* Must be able to pay for its own additional weight */
-	/* Rounding can cause off by one errors, so we do this */
-	if (!amount_sat_sub(&fee,
-			    amount_tx_fee(feerate_perkw, change_weight() + total_weight),
-			    amount_tx_fee(feerate_perkw, total_weight)))
-		abort();
-	return fee;
+	/* witness[0] = ""
+	 * witness[1] = sig
+	 * witness[2] = sig
+	 * witness[3] = 2 key key 2 CHECKMULTISIG
+	 */
+	return 1 + (1 + 0) + (1 + 72) + (1 + 72) + (1 + 1 + 33 + 33 + 1 + 1);
 }
 
 struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw,
 				size_t total_weight)
 {
-	struct amount_sat fee = change_fee(feerate_perkw, total_weight);
+	size_t outweight;
+	struct amount_sat change_fee;
 
-	if (!amount_sat_sub(&excess, excess, fee))
+	/* Must be able to pay for its own additional weight */
+	outweight = bitcoin_tx_output_weight(BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN);
+
+	/* Rounding can cause off by one errors, so we do this */
+	if (!amount_sat_sub(&change_fee,
+			    amount_tx_fee(feerate_perkw, outweight + total_weight),
+			    amount_tx_fee(feerate_perkw, total_weight)))
+		return AMOUNT_SAT(0);
+
+	if (!amount_sat_sub(&excess, excess, change_fee))
 		return AMOUNT_SAT(0);
 
 	/* Must be non-dust */
@@ -993,18 +918,3 @@ struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw,
 
 	return excess;
 }
-
-u32 tx_feerate(const struct bitcoin_tx *tx)
-{
-
-	u32 feerate;
-
-	/* Fee should not overflow! */
-	if (!amount_feerate(&feerate,
-			    bitcoin_tx_compute_fee(tx),
-			    bitcoin_tx_weight(tx))) {
-		abort();
-	}
-
-	return feerate;
-}

bitcoin/tx.h

@@ -14,9 +14,7 @@
 /* BIP 125: Any nsequence < 0xFFFFFFFE is replacable.
  * And bitcoind uses this value. */
 #define BITCOIN_TX_RBF_SEQUENCE 0xFFFFFFFD
-
 struct wally_psbt;
-struct ripemd160;
 
 struct bitcoin_txid {
 	struct sha256_double shad;
@@ -48,17 +46,6 @@ struct bitcoin_tx_output {
 	u8 *script;
 };
 
-enum utxotype {
-	/* Obsolete: we used to have P2SH-wrapped outputs (removed in 24.02, though can still have old UTXOs) */
-	UTXO_P2SH_P2WPKH = 1,
-	/* "bech32" addresses */
-	UTXO_P2WPKH = 2,
-	/* Used for closing addresses: implies ->close_info is non-NULL */
-	UTXO_P2WSH_FROM_CLOSE = 3,
-	/* "p2tr" addresses. */
-	UTXO_P2TR = 4,
-};
-
 struct bitcoin_tx_output *new_tx_output(const tal_t *ctx,
 					struct amount_sat amount,
 					const u8 *script);
@@ -82,26 +69,10 @@ struct bitcoin_tx *bitcoin_tx(const tal_t *ctx,
 			      varint_t input_count, varint_t output_count,
 			      u32 nlocktime);
 
-/* Make a (deep) copy */
-struct bitcoin_tx *clone_bitcoin_tx(const tal_t *ctx,
-				    const struct bitcoin_tx *tx TAKES);
-
 /* This takes a raw bitcoin tx in hex. */
 struct bitcoin_tx *bitcoin_tx_from_hex(const tal_t *ctx, const char *hex,
 				       size_t hexlen);
 
-/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
-static inline void reverse_bytes(u8 *arr, size_t len)
-{
-	unsigned int i;
-
-	for (i = 0; i < len / 2; i++) {
-		unsigned char tmp = arr[i];
-		arr[i] = arr[len - 1 - i];
-		arr[len - 1 - i] = tmp;
-	}
-}
-
 /* Parse hex string to get txid (reversed, a-la bitcoind). */
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid);
@@ -111,18 +82,12 @@ bool bitcoin_txid_to_hex(const struct bitcoin_txid *txid,
 			 char *hexstr, size_t hexstr_len);
 
 /* Create a bitcoin_tx from a psbt */
-struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx,
-					struct wally_psbt *psbt TAKES);
+struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt);
 
 /* Internal de-linearization functions. */
-/* Pull a bitcoin tx, and create a PSBT wrapper for it */
 struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx,
 				   const u8 **cursor, size_t *max);
 
-/* Pull a bitcoin tx without creating a PSBT wrapper for it */
-struct bitcoin_tx *pull_bitcoin_tx_only(const tal_t *ctx,
-					const u8 **cursor, size_t *max);
-
 /* Helper to create a wally_tx_output: make sure to wally_tx_output_free!
  * Returns NULL if amount is extreme (wally doesn't like).
  */
@@ -135,9 +100,6 @@ int bitcoin_tx_add_output(struct bitcoin_tx *tx, const u8 *script,
 			  const u8 *wscript,
 			  struct amount_sat amount);
 
-/* Remove one output. */
-void bitcoin_tx_remove_output(struct bitcoin_tx *tx, size_t outnum);
-
 /* Set the locktime for a transaction */
 void bitcoin_tx_set_locktime(struct bitcoin_tx *tx, u32 locktime);
 
@@ -170,6 +132,25 @@ wally_tx_output_get_amount(const struct wally_tx_output *output);
 void bitcoin_tx_output_set_amount(struct bitcoin_tx *tx, int outnum,
 				  struct amount_sat amount);
 
+/**
+ * Helper to get the script of a script's output as a tal_arr
+ *
+ * Internally we use a `wally_tx` to represent the transaction. The script
+ * attached to a `wally_tx_output` is not a `tal_arr`, so in order to keep the
+ * comfort of being able to call `tal_bytelen` and similar on a script we just
+ * return a `tal_arr` clone of the original script.
+ */
+const u8 *bitcoin_tx_output_get_script(const tal_t *ctx, const struct bitcoin_tx *tx, int outnum);
+
+/**
+ * Helper to get the script of a script's output as a tal_arr
+ *
+ * The script attached to a `wally_tx_output` is not a `tal_arr`, so in order to keep the
+ * comfort of being able to call `tal_bytelen` and similar on a script we just
+ * return a `tal_arr` clone of the original script.
+ */
+const u8 *wally_tx_output_get_script(const tal_t *ctx,
+				     const struct wally_tx_output *output);
 /**
  * Helper to get a witness script for an output.
  */
@@ -285,12 +266,6 @@ static inline size_t elements_tx_overhead(const struct chainparams *chainparams,
  */
 struct amount_sat bitcoin_tx_compute_fee(const struct bitcoin_tx *tx);
 
-/**
- * Calculate the feerate for this transaction (in perkw)
-*/
-u32 tx_feerate(const struct bitcoin_tx *tx);
-
-
 /*
  * Calculate the fees for this transaction, given a pre-computed input balance.
  *
@@ -310,15 +285,6 @@ void towire_bitcoin_tx(u8 **pptr, const struct bitcoin_tx *tx);
 void towire_bitcoin_outpoint(u8 **pptr, const struct bitcoin_outpoint *outp);
 void fromwire_bitcoin_outpoint(const u8 **cursor, size_t *max,
 			       struct bitcoin_outpoint *outp);
-char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx);
-char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid);
-char *fmt_bitcoin_outpoint(const tal_t *ctx,
-			   const struct bitcoin_outpoint *outpoint);
-char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx);
-
-/* For want of somewhere better to define them! */
-char *fmt_sha256(const tal_t *ctx, const struct sha256 *sha256);
-char *fmt_ripemd160(const tal_t *ctx, const struct ripemd160 *ripemd160);
 
 /* Various weights of transaction parts. */
 size_t bitcoin_tx_core_weight(size_t num_inputs, size_t num_outputs);
@@ -327,43 +293,29 @@ size_t bitcoin_tx_output_weight(size_t outscript_len);
 /* Weight to push sig on stack. */
 size_t bitcoin_tx_input_sig_weight(void);
 
-/* Segwit input, but with parameter for witness weight (size).
- * witness_weight must include the varint_size() for each witness element,
- * but not the varint_size() for the number of elements. */
+/* Segwit input, but with parameter for witness weight (size) */
 size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight);
 
-/* The witness weight */
-size_t bitcoin_tx_input_witness_weight(enum utxotype utxotype);
+/* The witness weight for a simple (sig + key) input */
+size_t bitcoin_tx_simple_input_witness_weight(void);
+
+/* We only do segwit inputs, and we assume witness is sig + key  */
+size_t bitcoin_tx_simple_input_weight(bool p2sh);
 
 /* The witness for our 2of2 input (closing or commitment tx). */
 size_t bitcoin_tx_2of2_input_witness_weight(void);
 
 /**
- * change_weight - what's the weight of a change output?
- */
-size_t change_weight(void);
-
-/**
- * change_fee - what's the cost to add a change output to this tx?
- * @feerate_perkw: feerate.
- * @total_weight: current weight of tx.
- *
- * We pass in the total_weight of the tx (up until this point) so as
- * to avoid any off-by-one errors with rounding the change fee (down)
- */
-struct amount_sat change_fee(u32 feerate_perkw,	size_t total_weight);
-
-/**
- * change_amount - Is it worth making a change output at this feerate?
+ * change_amount - Is it worth making a P2WPKH change output at this feerate?
  * @excess: input amount we have above the tx fee and other outputs.
  * @feerate_perkw: feerate.
- * @total_weight: current weight of tx.
- *
- * Change script is P2TR for Bitcoin, P2WPKH for Elements
  *
  * If it's not worth (or possible) to make change, returns AMOUNT_SAT(0).
  * Otherwise returns the amount of the change output to add (@excess minus
- * the change_fee()).
+ * the additional fee for the change output itself).
+ *
+ * We pass in the total_weight of the tx (up until this point) so as
+ * to avoid any off-by-one errors with rounding the change fee (down)
  */
 struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw,
 				size_t total_weight);

bitcoin/tx_parts.c

@@ -10,11 +10,69 @@ static void destroy_wally_tx_input(struct wally_tx_input *in)
 	wally_tx_input_free(in);
 }
 
+static struct wally_tx_input *clone_input(const struct wally_tx_input *src)
+{
+	struct wally_tx_input *in;
+	int ret;
+
+	if (is_elements(chainparams)) {
+		ret = wally_tx_elements_input_init_alloc
+			(src->txhash, sizeof(src->txhash),
+			 src->index, src->sequence,
+			 src->script, src->script_len,
+			 src->witness,
+			 src->blinding_nonce, sizeof(src->blinding_nonce),
+			 src->entropy, sizeof(src->entropy),
+			 src->issuance_amount, src->issuance_amount_len,
+			 src->inflation_keys, src->inflation_keys_len,
+			 src->issuance_amount_rangeproof,
+			 src->issuance_amount_rangeproof_len,
+			 src->inflation_keys_rangeproof,
+			 src->inflation_keys_rangeproof_len,
+			 src->pegin_witness,
+			 &in);
+	} else {
+		ret = wally_tx_input_init_alloc(src->txhash, sizeof(src->txhash),
+						src->index, src->sequence,
+						src->script, src->script_len,
+						src->witness, &in);
+	}
+	assert(ret == WALLY_OK);
+
+	tal_add_destructor(in, destroy_wally_tx_input);
+	return in;
+}
+
 static void destroy_wally_tx_output(struct wally_tx_output *out)
 {
 	wally_tx_output_free(out);
 }
 
+static struct wally_tx_output *clone_output(const struct wally_tx_output *src)
+{
+	struct wally_tx_output *out;
+	int ret;
+
+	if (is_elements(chainparams)) {
+		ret = wally_tx_elements_output_init_alloc
+			(src->script, src->script_len,
+			 src->asset, src->asset_len,
+			 src->value, src->value_len,
+			 src->nonce, src->nonce_len,
+			 src->surjectionproof, src->surjectionproof_len,
+			 src->rangeproof, src->rangeproof_len,
+			 &out);
+	} else {
+		ret = wally_tx_output_init_alloc(src->satoshi,
+						 src->script, src->script_len,
+						 &out);
+	}
+	assert(ret == WALLY_OK);
+
+	tal_add_destructor(out, destroy_wally_tx_output);
+	return out;
+}
+
 struct tx_parts *tx_parts_from_wally_tx(const tal_t *ctx,
 					const struct wally_tx *wtx,
 					int input, int output)
@@ -29,19 +87,13 @@ struct tx_parts *tx_parts_from_wally_tx(const tal_t *ctx,
 	for (size_t i = 0; i < wtx->num_inputs; i++) {
 		if (input != -1 && input != i)
 			continue;
-		if (wally_tx_input_clone_alloc(&wtx->inputs[i],
-					       &txp->inputs[i]) != WALLY_OK)
-			abort();
-		tal_add_destructor(txp->inputs[i], destroy_wally_tx_input);
+		txp->inputs[i] = clone_input(&wtx->inputs[i]);
 	}
 
 	for (size_t i = 0; i < wtx->num_outputs; i++) {
 		if (output != -1 && output != i)
 			continue;
-		if (wally_tx_output_clone_alloc(&wtx->outputs[i],
-						&txp->outputs[i]) != WALLY_OK)
-			abort();
-		tal_add_destructor(txp->outputs[i], destroy_wally_tx_output);
+		txp->outputs[i] = clone_output(&wtx->outputs[i]);
 
 		/* Cheat a bit by also setting the numeric satoshi
 		 * value, otherwise we end up converting a

ccan/README

@@ -1,3 +1,3 @@
 CCAN imported from http://ccodearchive.net.
 
-CCAN version: init-2593-gca094039
+CCAN version: init-2548-gab87e56b

ccan/ccan/base64/base64.c

@@ -31,7 +31,7 @@ static int8_t sixbit_from_b64(const base64_maps_t *maps,
 	int8_t ret;
 
 	ret = maps->decode_map[(unsigned char)b64letter];
-	if (ret == (int8_t)'\xff') {
+	if (ret == (char)0xff) {
 		errno = EDOM;
 		return -1;
 	}
@@ -41,7 +41,7 @@ static int8_t sixbit_from_b64(const base64_maps_t *maps,
 
 bool base64_char_in_alphabet(const base64_maps_t *maps, const char b64char)
 {
-	return (maps->decode_map[(const unsigned char)b64char] != (signed char)'\xff');
+	return (maps->decode_map[(const unsigned char)b64char] != (char)0xff);
 }
 
 void base64_init_maps(base64_maps_t *dest, const char src[64])

ccan/ccan/bitmap/bitmap.c

@@ -9,7 +9,7 @@
 #define BIT_ALIGN_DOWN(n)	((n) & ~(BITMAP_WORD_BITS - 1))
 #define BIT_ALIGN_UP(n)		BIT_ALIGN_DOWN((n) + BITMAP_WORD_BITS - 1)
 
-void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m)
+void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
 	unsigned long am = BIT_ALIGN_DOWN(m);
@@ -19,22 +19,22 @@ void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m)
 	assert(m >= n);
 
 	if (am < an) {
-		BITMAP_WORD(b, n) &= ~bitmap_bswap(headmask & tailmask);
+		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask & tailmask);
 		return;
 	}
 
 	if (an > n)
-		BITMAP_WORD(b, n) &= ~bitmap_bswap(headmask);
+		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask);
 
 	if (am > an)
-		memset(&BITMAP_WORD(b, an), 0,
+		memset(&BITMAP_WORD(bitmap, an), 0,
 		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
 
 	if (m > am)
-		BITMAP_WORD(b, m) &= ~bitmap_bswap(tailmask);
+		BITMAP_WORD(bitmap, m) &= ~bitmap_bswap(tailmask);
 }
 
-void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m)
+void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
 	unsigned long am = BIT_ALIGN_DOWN(m);
@@ -44,19 +44,19 @@ void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m)
 	assert(m >= n);
 
 	if (am < an) {
-		BITMAP_WORD(b, n) |= bitmap_bswap(headmask & tailmask);
+		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask & tailmask);
 		return;
 	}
 
 	if (an > n)
-		BITMAP_WORD(b, n) |= bitmap_bswap(headmask);
+		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask);
 
 	if (am > an)
-		memset(&BITMAP_WORD(b, an), 0xff,
+		memset(&BITMAP_WORD(bitmap, an), 0xff,
 		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
 
 	if (m > am)
-		BITMAP_WORD(b, m) |= bitmap_bswap(tailmask);
+		BITMAP_WORD(bitmap, m) |= bitmap_bswap(tailmask);
 }
 
 static int bitmap_clz(bitmap_word w)
@@ -76,7 +76,7 @@ static int bitmap_clz(bitmap_word w)
 #endif
 }
 
-unsigned long bitmap_ffs(const bitmap *b,
+unsigned long bitmap_ffs(const bitmap *bitmap,
 			 unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
@@ -87,7 +87,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	assert(m >= n);
 
 	if (am < an) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, n));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
 
 		w &= (headmask & tailmask);
 
@@ -95,7 +95,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	if (an > n) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, n));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
 
 		w &= headmask;
 
@@ -104,7 +104,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	while (an < am) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, an));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, an));
 
 		if (w)
 			return an + bitmap_clz(w);
@@ -113,7 +113,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	if (m > am) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, m));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, m));
 
 		w &= tailmask;
 

ccan/ccan/bitmap/bitmap.h

@@ -58,37 +58,37 @@ static inline bitmap_word bitmap_bswap(bitmap_word w)
 #define BITMAP_TAIL(_bm, _nbits) \
 	(BITMAP_TAILWORD(_bm, _nbits) & BITMAP_TAILBITS(_nbits))
 
-static inline void bitmap_set_bit(bitmap *b, unsigned long n)
+static inline void bitmap_set_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) |= BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) |= BITMAP_WORDBIT(n);
 }
 
-static inline void bitmap_clear_bit(bitmap *b, unsigned long n)
+static inline void bitmap_clear_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) &= ~BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) &= ~BITMAP_WORDBIT(n);
 }
 
-static inline void bitmap_change_bit(bitmap *b, unsigned long n)
+static inline void bitmap_change_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) ^= BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) ^= BITMAP_WORDBIT(n);
 }
 
-static inline bool bitmap_test_bit(const bitmap *b, unsigned long n)
+static inline bool bitmap_test_bit(const bitmap *bitmap, unsigned long n)
 {
-	return !!(BITMAP_WORD(b, n) & BITMAP_WORDBIT(n));
+	return !!(BITMAP_WORD(bitmap, n) & BITMAP_WORDBIT(n));
 }
 
-void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m);
-void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m);
+void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m);
+void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m);
 
-static inline void bitmap_zero(bitmap *b, unsigned long nbits)
+static inline void bitmap_zero(bitmap *bitmap, unsigned long nbits)
 {
-	memset(b, 0, bitmap_sizeof(nbits));
+	memset(bitmap, 0, bitmap_sizeof(nbits));
 }
 
-static inline void bitmap_fill(bitmap *b, unsigned long nbits)
+static inline void bitmap_fill(bitmap *bitmap, unsigned long nbits)
 {
-	memset(b, 0xff, bitmap_sizeof(nbits));
+	memset(bitmap, 0xff, bitmap_sizeof(nbits));
 }
 
 static inline void bitmap_copy(bitmap *dst, const bitmap *src,
@@ -161,36 +161,37 @@ static inline bool bitmap_subset(const bitmap *src1, const bitmap *src2,
 	return true;
 }
 
-static inline bool bitmap_full(const bitmap *b, unsigned long nbits)
+static inline bool bitmap_full(const bitmap *bitmap, unsigned long nbits)
 {
 	unsigned long i;
 
 	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
-		if (b[i].w != -1UL)
+		if (bitmap[i].w != -1UL)
 			return false;
 	}
 	if (BITMAP_HASTAIL(nbits) &&
-	    (BITMAP_TAIL(b, nbits) != BITMAP_TAILBITS(nbits)))
+	    (BITMAP_TAIL(bitmap, nbits) != BITMAP_TAILBITS(nbits)))
 		return false;
 
 	return true;
 }
 
-static inline bool bitmap_empty(const bitmap *b, unsigned long nbits)
+static inline bool bitmap_empty(const bitmap *bitmap, unsigned long nbits)
 {
 	unsigned long i;
 
 	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
-		if (b[i].w != 0)
+		if (bitmap[i].w != 0)
 			return false;
 	}
-	if (BITMAP_HASTAIL(nbits) && (BITMAP_TAIL(b, nbits) != 0))
+	if (BITMAP_HASTAIL(nbits) && (BITMAP_TAIL(bitmap, nbits) != 0))
 		return false;
 
 	return true;
 }
 
-unsigned long bitmap_ffs(const bitmap *b, unsigned long n, unsigned long m);
+unsigned long bitmap_ffs(const bitmap *bitmap,
+			 unsigned long n, unsigned long m);
 
 /*
  * Allocation functions
@@ -220,26 +221,26 @@ static inline bitmap *bitmap_alloc1(unsigned long nbits)
 	return bitmap;
 }
 
-static inline bitmap *bitmap_realloc0(bitmap *b,
+static inline bitmap *bitmap_realloc0(bitmap *bitmap,
 				      unsigned long obits, unsigned long nbits)
 {
-	b = realloc(b, bitmap_sizeof(nbits));
+	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
 
-	if ((nbits > obits) && b)
-		bitmap_zero_range(b, obits, nbits);
+	if ((nbits > obits) && bitmap)
+		bitmap_zero_range(bitmap, obits, nbits);
 
-	return b;
+	return bitmap;
 }
 
-static inline bitmap *bitmap_realloc1(bitmap *b,
+static inline bitmap *bitmap_realloc1(bitmap *bitmap,
 				      unsigned long obits, unsigned long nbits)
 {
-	b = realloc(b, bitmap_sizeof(nbits));
+	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
 
-	if ((nbits > obits) && b)
-		bitmap_fill_range(b, obits, nbits);
+	if ((nbits > obits) && bitmap)
+		bitmap_fill_range(bitmap, obits, nbits);
 
-	return b;
+	return bitmap;
 }
 
 #endif /* CCAN_BITMAP_H_ */

ccan/ccan/bitops/test/run.c

@@ -10,7 +10,7 @@ int main(void)
 	plan_tests(68 + 6 * (31 + 63));
 
 	for (i = 0; i < 32; i++)
-		ok1(bitops_ffs32(1U << i) == i+1);
+		ok1(bitops_ffs32(1 << i) == i+1);
 	ok1(bitops_ffs32(0) == 0);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_ffs64((uint64_t)1 << i) == i+1);
@@ -25,19 +25,19 @@ int main(void)
 	ok1(bitops_ffs64(0) == 0);
 
 	for (i = 0; i < 32; i++)
-		ok1(bitops_clz32(1U << i) == 31 - i);
+		ok1(bitops_clz32(1 << i) == 31 - i);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_clz64((uint64_t)1 << i) == 63 - i);
 
 	/* Lower bits don't effect results */
 	for (i = 0; i < 32; i++)
-		ok1(bitops_clz32((1U << i) + (1U << i)-1) == 31 - i);
+		ok1(bitops_clz32((1 << i) + (1 << i)-1) == 31 - i);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_clz64(((uint64_t)1 << i) + ((uint64_t)1 << i)-1)
 		    == 63 - i);
 
 	for (i = 0; i < 32; i++)
-		ok1(bitops_ctz32(1U << i) == i);
+		ok1(bitops_ctz32(1 << i) == i);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_ctz64((uint64_t)1 << i) == i);
 

ccan/ccan/crypto/hmac_sha256/hmac_sha256.c

@@ -35,8 +35,7 @@ void hmac_sha256_init(struct hmac_sha256_ctx *ctx,
 	 *  (e.g., if K is of length 20 bytes and B=64, then K will be
 	 *   appended with 44 zero bytes 0x00)
 	 */
-	if (ksize != 0)
-		memcpy(k_ipad, k, ksize);
+	memcpy(k_ipad, k, ksize);
 	memset((char *)k_ipad + ksize, 0, HMAC_SHA256_BLOCKSIZE - ksize);
 
 	/*

ccan/ccan/fdpass/fdpass.c

@@ -3,7 +3,6 @@
 #include <sys/socket.h>
 #include <errno.h>
 #include <string.h>
-#include <sys/types.h>
 
 bool fdpass_send(int sockout, int fd)
 {

ccan/ccan/htable/htable_type.h

@@ -1,23 +1,19 @@
 /* Licensed under LGPLv2+ - see LICENSE file for details */
 #ifndef CCAN_HTABLE_TYPE_H
 #define CCAN_HTABLE_TYPE_H
-#include "config.h"
-#include <assert.h>
 #include <ccan/htable/htable.h>
 #include <ccan/compiler/compiler.h>
+#include "config.h"
 
 /**
- * HTABLE_DEFINE_NODUPS_TYPE/HTABLE_DEFINE_DUPS_TYPE - create a set of htable ops for a type
+ * HTABLE_DEFINE_TYPE - create a set of htable ops for a type
  * @type: a type whose pointers will be values in the hash.
  * @keyof: a function/macro to extract a key: <keytype> @keyof(const type *elem)
  * @hashfn: a hash function for a @key: size_t @hashfn(const <keytype> *)
  * @eqfn: an equality function keys: bool @eqfn(const type *, const <keytype> *)
  * @prefix: a prefix for all the functions to define (of form <name>_*)
  *
- * There are two variants, one of which allows duplicate keys, and one which
- * does not.  The defined functions differ in some cases, as shown below.
- *
- * NULL values may not be placed into the hash table (nor (void *)1).
+ * NULL values may not be placed into the hash table.
  *
  * This defines the type hashtable type and an iterator type:
  *	struct <name>;
@@ -37,18 +33,15 @@
  *
  * Delete and delete-by key return true if it was in the set:
  *	bool <name>_del(struct <name> *ht, const <type> *e);
- *	bool <name>_delkey(struct <name> *ht, const <keytype> *k) (NODUPS only);
+ *	bool <name>_delkey(struct <name> *ht, const <keytype> *k);
  *
  * Delete by iterator:
  *	bool <name>_delval(struct <name> *ht, struct <name>_iter *i);
  *
- * Find and return the matching element, or NULL:
- *	type *<name>_get(const struct @name *ht, const <keytype> *k) (NODUPS only);
+ * Find and return the (first) matching element, or NULL:
+ *	type *<name>_get(const struct @name *ht, const <keytype> *k);
  *
- * Test for an element:
- *	bool <name>_exists(const struct @name *ht, const <keytype> *k);
- *
- * Find and return all matching elements, or NULL (DUPS only):
+ * Find and return all matching elements, or NULL:
  *	type *<name>_getfirst(const struct @name *ht, const <keytype> *k,
  *			      struct <name>_iter *i);
  *	type *<name>_getnext(const struct @name *ht, const <keytype> *k,
@@ -66,7 +59,7 @@
  * You can use HTABLE_INITIALIZER like so:
  *	struct <name> ht = { HTABLE_INITIALIZER(ht.raw, <name>_hash, NULL) };
  */
-#define HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
+#define HTABLE_DEFINE_TYPE(type, keyof, hashfn, eqfn, name)		\
 	struct name { struct htable raw; };				\
 	struct name##_iter { struct htable_iter i; };			\
 	static inline size_t name##_hash(const void *elem, void *priv)	\
@@ -96,33 +89,66 @@
 	{								\
 		return htable_copy(&dst->raw, &src->raw);		\
 	}								\
+	static inline bool name##_add(struct name *ht, const type *elem) \
+	{								\
+		return htable_add(&ht->raw, hashfn(keyof(elem)), elem);	\
+	}								\
 	static inline UNNEEDED bool name##_del(struct name *ht,		\
 					       const type *elem)	\
 	{								\
 		return htable_del(&ht->raw, hashfn(keyof(elem)), elem);	\
 	}								\
-	static inline UNNEEDED type *name##_getmatch_(const struct name *ht, \
-				         const HTABLE_KTYPE(keyof, type) k, \
-				         size_t h,			\
-				         type *v,			\
-					 struct htable_iter *iter)	\
-	{								\
-		while (v) {						\
-			if (eqfn(v, k))					\
-				break;					\
-			v = htable_nextval(&ht->raw, iter, h);		\
-		}							\
-		return v;						\
-	}								\
-	static inline UNNEEDED bool name##_exists(const struct name *ht,	\
+	static inline UNNEEDED type *name##_get(const struct name *ht,	\
 				       const HTABLE_KTYPE(keyof, type) k) \
 	{								\
 		struct htable_iter i;					\
 		size_t h = hashfn(k);					\
-		void *v;						\
+		void *c;						\
 									\
-		v = htable_firstval(&ht->raw, &i, h);			\
-		return name##_getmatch_(ht, k, h, v, &i) != NULL;	\
+		for (c = htable_firstval(&ht->raw,&i,h);		\
+		     c;							\
+		     c = htable_nextval(&ht->raw,&i,h)) {		\
+			if (eqfn(c, k))					\
+				return c;				\
+		}							\
+		return NULL;						\
+	}								\
+	static inline UNNEEDED type *name##_getmatch_(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+				         size_t h,			\
+				         type *v,			\
+					 struct name##_iter *iter)	\
+	{								\
+		while (v) {						\
+			if (eqfn(v, k))					\
+				break;					\
+			v = htable_nextval(&ht->raw, &iter->i, h);	\
+		}							\
+		return v;						\
+	}								\
+	static inline UNNEEDED type *name##_getfirst(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+					 struct name##_iter *iter)	\
+	{								\
+		size_t h = hashfn(k);					\
+		type *v = htable_firstval(&ht->raw, &iter->i, h);	\
+		return name##_getmatch_(ht, k, h, v, iter);			\
+	}								\
+	static inline UNNEEDED type *name##_getnext(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+					 struct name##_iter *iter)	\
+	{								\
+		size_t h = hashfn(k);					\
+		type *v = htable_nextval(&ht->raw, &iter->i, h);	\
+		return name##_getmatch_(ht, k, h, v, iter);		\
+	}								\
+	static inline UNNEEDED bool name##_delkey(struct name *ht,	\
+					 const HTABLE_KTYPE(keyof, type) k) \
+	{								\
+		type *elem = name##_get(ht, k);				\
+		if (elem)						\
+			return name##_del(ht, elem);			\
+		return false;						\
 	}								\
 	static inline UNNEEDED void name##_delval(struct name *ht,	\
 						  struct name##_iter *iter) \
@@ -133,7 +159,8 @@
 						size_t seed,		\
 						struct name##_iter *iter) \
 	{								\
-		return htable_pick(&ht->raw, seed, iter ? &iter->i : NULL); \
+		/* Note &iter->i == NULL iff iter is NULL */		\
+		return htable_pick(&ht->raw, seed, &iter->i);			\
 	}								\
 	static inline UNNEEDED type *name##_first(const struct name *ht, \
 					 struct name##_iter *iter)	\
@@ -151,64 +178,6 @@
 		return htable_prev(&ht->raw, &iter->i);			\
 	}
 
-#define HTABLE_DEFINE_NODUPS_TYPE(type, keyof, hashfn, eqfn, name)	\
-	HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
-	static inline UNNEEDED type *name##_get(const struct name *ht,	\
-				       const HTABLE_KTYPE(keyof, type) k) \
-	{								\
-		struct htable_iter i;					\
-		size_t h = hashfn(k);					\
-		void *v;						\
-									\
-		v = htable_firstval(&ht->raw, &i, h);			\
-		return name##_getmatch_(ht, k, h, v, &i);			\
-	}								\
-	static inline bool name##_add(struct name *ht, const type *elem) \
-	{								\
-		/* Open-coded for slightly more efficiency */		\
-		const HTABLE_KTYPE(keyof, type) k = keyof(elem);	\
-		struct htable_iter i;					\
-		size_t h = hashfn(k);					\
-		void *v;						\
-									\
-		v = htable_firstval(&ht->raw, &i, h);			\
-		assert(!name##_getmatch_(ht, k, h, v, &i));		\
-		return htable_add(&ht->raw, h, elem);			\
-	}								\
-	static inline UNNEEDED bool name##_delkey(struct name *ht,	\
-					 const HTABLE_KTYPE(keyof, type) k) \
-	{								\
-		type *elem = name##_get(ht, k);				\
-		if (elem)						\
-			return name##_del(ht, elem);			\
-		return false;						\
-	}
-
-#define HTABLE_DEFINE_DUPS_TYPE(type, keyof, hashfn, eqfn, name)	\
-	HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
-	static inline bool name##_add(struct name *ht, const type *elem) \
-	{								\
-		const HTABLE_KTYPE(keyof, type) k = keyof(elem);	\
-		return htable_add(&ht->raw, hashfn(k), elem);		\
-	}								\
-	static inline UNNEEDED type *name##_getfirst(const struct name *ht, \
-				         const HTABLE_KTYPE(keyof, type) k, \
-					 struct name##_iter *iter)	\
-	{								\
-		size_t h = hashfn(k);					\
-		type *v = htable_firstval(&ht->raw, &iter->i, h);	\
-		return name##_getmatch_(ht, k, h, v, &iter->i);		\
-	}								\
-	static inline UNNEEDED type *name##_getnext(const struct name *ht, \
-				         const HTABLE_KTYPE(keyof, type) k, \
-					 struct name##_iter *iter)	\
-	{								\
-		size_t h = hashfn(k);					\
-		type *v = htable_nextval(&ht->raw, &iter->i, h);	\
-		return name##_getmatch_(ht, k, h, v, &iter->i);		\
-	}
-
-
 #if HAVE_TYPEOF
 #define HTABLE_KTYPE(keyof, type) typeof(keyof((const type *)NULL))
 #else

ccan/ccan/htable/test/run-type-int.c

@@ -38,7 +38,7 @@ static bool cmp(const struct obj *obj, const unsigned int key)
 	return obj->key == key;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
+HTABLE_DEFINE_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
 
 static void add_vals(struct htable_obj *ht,
 		     struct obj val[], unsigned int num)
@@ -112,19 +112,14 @@ static bool check_mask(struct htable *ht, const struct obj val[], unsigned num)
 	return true;
 }
 
-/* This variant allows duplicates! */
-HTABLE_DEFINE_DUPS_TYPE(struct obj, objkey, objhash, cmp, htable_obj_dups);
-
 int main(void)
 {
 	unsigned int i;
 	struct htable_obj ht, ht2;
-	struct htable_obj_dups ht_dups;
 	struct obj val[NUM_VALS], *result;
 	unsigned int dne;
 	void *p;
 	struct htable_obj_iter iter;
-	struct htable_obj_dups_iter dups_iter;
 
 	plan_tests(29);
 	for (i = 0; i < NUM_VALS; i++)
@@ -188,35 +183,32 @@ int main(void)
 	del_vals_bykey(&ht, val, NUM_VALS);
 	del_vals_bykey(&ht2, val, NUM_VALS);
 
-	/* Duplicates-allowed tests */
-	htable_obj_dups_init(&ht_dups);
 	/* Write two of the same value. */
 	val[1] = val[0];
-	htable_obj_dups_add(&ht_dups, &val[0]);
-	htable_obj_dups_add(&ht_dups, &val[1]);
+	htable_obj_add(&ht, &val[0]);
+	htable_obj_add(&ht, &val[1]);
 	i = 0;
 
-	result = htable_obj_dups_getfirst(&ht_dups, i, &dups_iter);
+	result = htable_obj_getfirst(&ht, i, &iter);
 	ok1(result == &val[0] || result == &val[1]);
 	if (result == &val[0]) {
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[0]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[0]));
+		ok1(htable_obj_getfirst(&ht, i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 	} else {
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[1]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[1]));
+		ok1(htable_obj_getfirst(&ht, i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 	}
 
-	htable_obj_dups_clear(&ht_dups);
 	htable_obj_clear(&ht);
 	htable_obj_clear(&ht2);
 	return exit_status();

ccan/ccan/htable/test/run-type.c

@@ -33,10 +33,7 @@ static bool cmp(const struct obj *obj, const unsigned int *key)
 	return obj->key == *key;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(struct obj, objkey, objhash, cmp,
-			  htable_obj);
-HTABLE_DEFINE_DUPS_TYPE(struct obj, objkey, objhash, cmp,
-			htable_obj_dups);
+HTABLE_DEFINE_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
 
 static void add_vals(struct htable_obj *ht,
 		     struct obj val[], unsigned int num)
@@ -114,14 +111,12 @@ int main(void)
 {
 	unsigned int i;
 	struct htable_obj ht, ht2;
-	struct htable_obj_dups ht_dups;
 	struct obj val[NUM_VALS], *result;
 	unsigned int dne;
 	void *p;
 	struct htable_obj_iter iter;
-	struct htable_obj_dups_iter dups_iter;
 
-	plan_tests(36);
+	plan_tests(35);
 	for (i = 0; i < NUM_VALS; i++)
 		val[i].key = i;
 	dne = i;
@@ -187,37 +182,32 @@ int main(void)
 	del_vals_bykey(&ht, val, NUM_VALS);
 	del_vals_bykey(&ht2, val, NUM_VALS);
 
-	/* Duplicates-allowed tests */
-	htable_obj_dups_init(&ht_dups);
-
 	/* Write two of the same value. */
 	val[1] = val[0];
-	htable_obj_dups_add(&ht_dups, &val[0]);
-	htable_obj_dups_add(&ht_dups, &val[1]);
+	htable_obj_add(&ht, &val[0]);
+	htable_obj_add(&ht, &val[1]);
 	i = 0;
 
-	result = htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter);
+	result = htable_obj_getfirst(&ht, &i, &iter);
 	ok1(result == &val[0] || result == &val[1]);
 	if (result == &val[0]) {
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[0]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[0]));
+		ok1(htable_obj_getfirst(&ht, &i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 	} else {
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[1]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[1]));
+		ok1(htable_obj_getfirst(&ht, &i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 	}
 
-	htable_obj_dups_clear(&ht_dups);
-	ok1(htable_obj_dups_count(&ht_dups) == 0);
 	htable_obj_clear(&ht);
 	ok1(htable_obj_count(&ht) == 0);
 	htable_obj_clear(&ht2);

ccan/ccan/htable/tools/density.c

@@ -26,7 +26,7 @@ static bool cmp(const ptrint_t *p, uintptr_t k)
 	return key(p) == k;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(ptrint_t, key, hash_uintptr, cmp, htable_ptrint);
+HTABLE_DEFINE_TYPE(ptrint_t, key, hash_uintptr, cmp, htable_ptrint);
 
 /* Nanoseconds per operation */
 static size_t normalize(const struct timeabs *start,

ccan/ccan/htable/tools/speed.c

@@ -33,7 +33,7 @@ static bool cmp(const struct object *object, const unsigned int *key)
 	return object->key == *key;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(struct object, objkey, hash_obj, cmp, htable_obj);
+HTABLE_DEFINE_TYPE(struct object, objkey, hash_obj, cmp, htable_obj);
 
 static unsigned int popcount(unsigned long val)
 {

ccan/ccan/htable/tools/stringspeed.c

@@ -31,7 +31,7 @@ static bool cmp(const char *obj, const char *key)
 	return strcmp(obj, key) == 0;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(char, strkey, hash_str, cmp, htable_str);
+HTABLE_DEFINE_TYPE(char, strkey, hash_str, cmp, htable_str);
 
 /* Nanoseconds per operation */
 static size_t normalize(const struct timeabs *start,

ccan/ccan/ilog/ilog.h

@@ -120,10 +120,7 @@ int ilog64_nz(uint64_t _v) CONST_FUNCTION;
 #endif
 
 #ifdef builtin_ilog32_nz
-/* This used to be builtin_ilog32_nz(_v)&-!!(_v), which means it zeroes out
- * the undefined builtin_ilog32_nz(0) return.  But clang UndefinedBehaviorSantizer
- * complains, so do the branch: */
-#define ilog32(_v) ((_v) ? builtin_ilog32_nz(_v) : 0)
+#define ilog32(_v) (builtin_ilog32_nz(_v)&-!!(_v))
 #define ilog32_nz(_v) builtin_ilog32_nz(_v)
 #else
 #define ilog32_nz(_v) ilog32(_v)
@@ -131,7 +128,7 @@ int ilog64_nz(uint64_t _v) CONST_FUNCTION;
 #endif /* builtin_ilog32_nz */
 
 #ifdef builtin_ilog64_nz
-#define ilog32(_v) ((_v) ? builtin_ilog32_nz(_v) : 0)
+#define ilog64(_v) (builtin_ilog64_nz(_v)&-!!(_v))
 #define ilog64_nz(_v) builtin_ilog64_nz(_v)
 #else
 #define ilog64_nz(_v) ilog64(_v)

ccan/ccan/intmap/benchmark/speed.c

@@ -59,7 +59,7 @@ static bool eqfn(const struct htable_elem *elem, const uint64_t index)
 {
 	return elem->index == index;
 }
-HTABLE_DEFINE_NODUPS_TYPE(struct htable_elem, keyof, hashfn, eqfn, hash);
+HTABLE_DEFINE_TYPE(struct htable_elem, keyof, hashfn, eqfn, hash);
 
 static bool check_val(intmap_index_t i, uint64_t *v, uint64_t *expected)
 {

ccan/ccan/io/benchmarks/Makefile

@@ -1,17 +1,16 @@
-ALL:=run-loop run-different-speed run-length-prefix run-stream-many
+ALL:=run-loop run-different-speed run-length-prefix
 CCANDIR:=../../..
 CFLAGS:=-Wall -I$(CCANDIR) -O3 -flto
 LDFLAGS:=-O3 -flto
 LDLIBS:=-lrt
 
-OBJS:=time.o poll.o io.o err.o timer.o list.o ccan-tal.o ccan-take.o ccan-ilog.o
+OBJS:=time.o poll.o io.o err.o timer.o list.o
 
 default: $(ALL)
 
 run-loop: run-loop.o $(OBJS)
 run-different-speed: run-different-speed.o $(OBJS)
 run-length-prefix: run-length-prefix.o $(OBJS)
-run-stream-many: run-stream-many.o $(OBJS)
 
 time.o: $(CCANDIR)/ccan/time/time.c
 	$(CC) $(CFLAGS) -c -o $@ $<
@@ -25,12 +24,6 @@ io.o: $(CCANDIR)/ccan/io/io.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 err.o: $(CCANDIR)/ccan/err/err.c
 	$(CC) $(CFLAGS) -c -o $@ $<
-ccan-ilog.o: $(CCANDIR)/ccan/ilog/ilog.c
-	$(CC) $(CFLAGS) -c -o $@ $<
-ccan-tal.o: $(CCANDIR)/ccan/tal/tal.c
-	$(CC) $(CFLAGS) -c -o $@ $<
-ccan-take.o: $(CCANDIR)/ccan/take/take.c
-	$(CC) $(CFLAGS) -c -o $@ $<
 
 clean:
 	rm -f *.o $(ALL)

ccan/ccan/io/benchmarks/run-stream-many.c

@@ -1,132 +0,0 @@
-/* Wait for many fds to connect, then try to stream the file to some of them in small chunks.
- *
- * This approximates the connectd behaviour in CLN, where we send gossip to peers.
- */
-#include <ccan/io/io.h>
-#include <ccan/ptrint/ptrint.h>
-#include <ccan/time/time.h>
-#include <inttypes.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-#include <err.h>
-#include <signal.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-
-/* We expect num_expected connections, and how many will be writers */
-static size_t max_readers, max_writers;
-
-/* How many raeaders and writers still going */
-static size_t num_readers, num_writers;
-
-/* How many times to do the write */
-static size_t write_iterations;
-
-/* The buffer to write */
-static char writebuf[256];
-
-/* We need this for readers, though we don't actually care! */
-static size_t len_ignored;
-
-struct timemono start_time;
-
-static void finished(void)
-{
-	struct timerel elapsed = timemono_since(start_time);
-	printf("Finished: %"PRIu64"usec\n", time_to_usec(elapsed));
-	exit(0);
-}
-
-static struct io_plan *write_loop(struct io_conn *conn, ptrint_t *iter)
-{
-	ptrdiff_t n = ptr2int(iter);
-
-	if (n > write_iterations) {
-		--num_writers;
-		if (num_writers == 0)
-			finished();
-		return io_wait(conn, conn, io_never, NULL);
-	}
-	return io_write(conn, writebuf, sizeof(writebuf), write_loop, int2ptr(n + 1));
-}
-
-static struct io_plan *read_loop(struct io_conn *conn, void *unused)
-{
-	return io_read_partial(conn, writebuf, sizeof(writebuf), &len_ignored, read_loop, unused);
-}
-
-static void reader_failed(struct io_conn *conn, intptr_t *num)
-{
-	err(1, "Reader %zu/%zu", (size_t)ptr2int(num), max_readers);
-}
-
-static void writer_failed(struct io_conn *conn, intptr_t *num)
-{
-	err(1, "Writer %zu/%zu", (size_t)ptr2int(num), max_writers);
-}
-
-static struct io_plan *connection_in(struct io_conn *conn, void *sleep_on)
-{
-	if (num_readers < max_readers) {
-		printf("r");
-		fflush(stdout);
-		num_readers++;
-		io_set_finish(conn, reader_failed, int2ptr(num_readers));
-		return read_loop(conn, NULL);
-	}
-
-	/* We assign writers last: not sure it matters, but it's more reflective
-	 * of lightning where more recent connections tend to ask for gossip */
-	num_writers++;
-	printf("w");
-	fflush(stdout);
-
-	io_set_finish(conn, writer_failed, int2ptr(num_writers));
-	io_set_finish(conn, writer_failed, NULL);
-	if (num_writers < max_writers)
-		return io_wait(conn, sleep_on, write_loop, int2ptr(0));
-
-	/* Everyone is connected.  Wake them and start final one */
-	io_wake(sleep_on);
-	printf("Starting!\n");
-	start_time = time_mono();
-	return write_loop(conn, int2ptr(0));
-}
-
-int main(int argc, char *argv[])
-{
-	int fd;
-	struct sockaddr_in s4;
-	int on = 1;
-
-	if (argc != 5)
-		errx(1, "Usage: <portnum> <num-idle> <num-streaming> <mb-streamed>");
-
-	memset(&s4, 0, sizeof(s4));
-	s4.sin_family = AF_INET;
-	s4.sin_port = htons(atol(argv[1]));
-	s4.sin_addr.s_addr = INADDR_ANY;
-
-	max_readers = atol(argv[2]);
-	max_writers = atol(argv[3]);
-	write_iterations = atol(argv[4]) * (1024 * 1024 / sizeof(writebuf));
-
-	fd = socket(AF_INET, SOCK_STREAM, 0);
-	if (fd < 0)
-		err(1, "Creating socket");
-
-	/* Re-use, please.. */
-	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)))
-		err(1, "Setting reuseaddr");
-
-	if (bind(fd, &s4, sizeof(s4)) != 0)
-		err(1, "Binding");
-
-	if (listen(fd, 1) != 0)
-		err(1, "Listening");
-
-	io_new_listener(NULL, fd, connection_in, &s4);
-	io_loop(NULL, NULL);
-	errx(1, "Sockets exited?");
-}

ccan/ccan/io/fdpass/_info

@@ -32,20 +32,12 @@
  *					read_more, buf);
  *	}
  *
- *	// Clean up allocation so -fsanitize=address doesn't see leak!
- *	static void free_buf(struct io_conn *c, struct buf *buf)
- *	{
- *		free(buf);
- *	}
- *
  *	// Child has received fd, start reading loop.
  *	static struct io_plan *got_infd(struct io_conn *conn, int *infd)
  *	{
  *		struct buf *buf = calloc(1, sizeof(*buf));
- *		struct io_conn *new_conn;
  *
- *		new_conn = io_new_conn(NULL, *infd, read_more, buf);
- *		io_set_finish(new_conn, free_buf, buf);
+ *		io_new_conn(NULL, *infd, read_more, buf);
  *		return io_close(conn);
  *	}
  *	// Child is receiving the fd to read into. 

ccan/ccan/io/fdpass/fdpass.c

@@ -4,12 +4,6 @@
 #include <ccan/io/io_plan.h>
 #include <errno.h>
 
-static void destroy_conn_close_send_fd(struct io_conn *conn,
-				       struct io_plan_arg *arg)
-{
-	close(arg->u1.s);
-}
-
 static int do_fd_send(int fd, struct io_plan_arg *arg)
 {
 	if (!fdpass_send(fd, arg->u1.s)) {
@@ -18,11 +12,8 @@ static int do_fd_send(int fd, struct io_plan_arg *arg)
 			return 0;
 		return -1;
 	}
-	if (arg->u2.vp) {
-		struct io_conn *conn = arg->u2.vp;
+	if (arg->u2.s)
 		close(arg->u1.s);
-		tal_del_destructor2(conn, destroy_conn_close_send_fd, arg);
-	}
 	return 1;
 }
 
@@ -35,11 +26,7 @@ struct io_plan *io_send_fd_(struct io_conn *conn,
 	struct io_plan_arg *arg = io_plan_arg(conn, IO_OUT);
 
 	arg->u1.s = fd;
-	/* We need conn ptr for destructor */
-	arg->u2.vp = fdclose ? conn : NULL;
-	/* If conn closes before sending, we still need to close fd */
-	if (fdclose)
-		tal_add_destructor2(conn, destroy_conn_close_send_fd, arg);
+	arg->u2.s = fdclose;
 
 	return io_set_plan(conn, IO_OUT, do_fd_send, next, next_arg);
 }
@@ -52,9 +39,7 @@ static int do_fd_recv(int fd, struct io_plan_arg *arg)
 		/* In case ccan/io ever gets smart with non-blocking. */
 		if (errno == EAGAIN || errno == EWOULDBLOCK)
 			return 0;
-		/* If they can't handle the error, this will close conn! */
-		if (!io_get_extended_errors())
-			return -1;
+		return -1;
 	}
 	*(int *)arg->u1.vp = fdin;
 	return 1;

ccan/ccan/io/fdpass/fdpass.h

@@ -45,9 +45,8 @@ struct io_plan *io_send_fd_(struct io_conn *conn,
  * @arg: @next argument
  *
  * This creates a plan to receive a file descriptor, as sent by
- * io_send_fd.  Once it's all read, the @next function will be called.
- * On an error, if io_get_extended_errors() is true, then @next is called
- * and @fd will be -1, otherwise the finish function is called.
+ * io_send_fd.  Once it's all read, the @next function will be called:
+ * on an error, the finish function is called instead.
  *
  * Note that the I/O may actually be done immediately.
  *

ccan/ccan/io/io.c

@@ -15,7 +15,6 @@
 void *io_loop_return;
 
 struct io_plan io_conn_freed;
-static bool io_extended_errors;
 
 struct io_listener *io_new_listener_(const tal_t *ctx, int fd,
 				     struct io_plan *(*init)(struct io_conn *,
@@ -384,19 +383,9 @@ void io_wake(const void *wait)
 	backend_wake(wait);
 }
 
-enum plan_result {
-	/* Destroyed, do not touch */
-	FREED,
-	/* Worked, call again. */
-	KEEP_GOING,
-	/* Failed with EAGAIN or did partial. */
-	EXHAUSTED,
-	/* No longer interested in read (or write) */
-	UNINTERESTED
-};
-
-static enum plan_result do_plan(struct io_conn *conn, struct io_plan *plan,
-				bool idle_on_epipe)
+/* Returns false if this should not be touched (eg. freed). */
+static bool do_plan(struct io_conn *conn, struct io_plan *plan,
+		    bool idle_on_epipe)
 {
 	/* We shouldn't have polled for this event if this wasn't true! */
 	assert(plan->status == IO_POLLING_NOTSTARTED
@@ -404,26 +393,18 @@ static enum plan_result do_plan(struct io_conn *conn, struct io_plan *plan,
 
 	switch (plan->io(conn->fd.fd, &plan->arg)) {
 	case -1:
-		/* This is expected, as we call optimistically! */
-		if (errno == EAGAIN)
-			return EXHAUSTED;
 		if (errno == EPIPE && idle_on_epipe) {
 			plan->status = IO_UNSET;
 			backend_new_plan(conn);
-			return UNINTERESTED;
+			return false;
 		}
 		io_close(conn);
-		return FREED;
+		return false;
 	case 0:
 		plan->status = IO_POLLING_STARTED;
-		/* If it started but didn't finish, don't call again. */
-		return EXHAUSTED;
+		return true;
 	case 1:
-		if (!next_plan(conn, plan))
-			return FREED;
-		if (plan->status == IO_POLLING_NOTSTARTED)
-			return KEEP_GOING;
-		return UNINTERESTED;
+		return next_plan(conn, plan);
 	default:
 		/* IO should only return -1, 0 or 1 */
 		abort();
@@ -432,43 +413,16 @@ static enum plan_result do_plan(struct io_conn *conn, struct io_plan *plan,
 
 void io_ready(struct io_conn *conn, int pollflags)
 {
-	enum plan_result res;
+	if (pollflags & POLLIN)
+		if (!do_plan(conn, &conn->plan[IO_IN], false))
+			return;
 
-	if (pollflags & POLLIN) {
-		for (;;) {
-			res = do_plan(conn, &conn->plan[IO_IN], false);
-			switch (res) {
-			case FREED:
-				return;
-			case EXHAUSTED:
-			case UNINTERESTED:
-				goto try_write;
-			case KEEP_GOING:
-				continue;
-			}
-			abort();
-		}
-	}
-
-try_write:
-	if (pollflags & POLLOUT) {
-		for (;;) {
-			/* If we're writing to a closed pipe, we need to wait for
-			 * read to fail if we're duplex: we want to drain it! */
-			res = do_plan(conn, &conn->plan[IO_OUT],
-				      conn->plan[IO_IN].status == IO_POLLING_NOTSTARTED
-				      || conn->plan[IO_IN].status == IO_POLLING_STARTED);
-			switch (res) {
-			case FREED:
-			case EXHAUSTED:
-			case UNINTERESTED:
-				return;
-			case KEEP_GOING:
-				continue;
-			}
-			abort();
-		}
-	}
+	if (pollflags & POLLOUT)
+		/* If we're writing to a closed pipe, we need to wait for
+		 * read to fail if we're duplex: we want to drain it! */
+		do_plan(conn, &conn->plan[IO_OUT],
+			conn->plan[IO_IN].status == IO_POLLING_NOTSTARTED
+			|| conn->plan[IO_IN].status == IO_POLLING_STARTED);
 }
 
 void io_do_always(struct io_plan *plan)
@@ -586,7 +540,7 @@ struct io_plan *io_sock_shutdown(struct io_conn *conn)
 	/* And leave unset .*/
 	return &conn->plan[IO_IN];
 }
-
+	
 bool io_flush_sync(struct io_conn *conn)
 {
 	struct io_plan *plan = &conn->plan[IO_OUT];
@@ -622,13 +576,3 @@ again:
 	io_fd_block(io_conn_fd(conn), false);
 	return ok;
 }
-
-void io_set_extended_errors(bool state)
-{
-	io_extended_errors = state;
-}
-
-bool io_get_extended_errors(void)
-{
-	return io_extended_errors;
-}

ccan/ccan/io/io.h

@@ -59,7 +59,7 @@ struct io_conn;
 	io_new_conn_((ctx), (fd),					\
 		     typesafe_cb_preargs(struct io_plan *, void *,	\
 					 (init), (arg),			\
-					 struct io_conn *),		\
+					 struct io_conn *conn),		\
 		     (void *)(arg))
 
 struct io_conn *io_new_conn_(const tal_t *ctx, int fd,
@@ -113,8 +113,6 @@ void io_set_finish_(struct io_conn *conn,
  * (tal'ocated off @ctx) and pass that to init().  Note that if there is
  * an error on this file descriptor, it will be freed.
  *
- * Note: if the accept fails (usually due to EMFILE), init() will be called
- * wth
  * Returns NULL on error (and sets errno).
  *
  * Example:
@@ -825,13 +823,4 @@ int (*io_poll_override(int (*poll)(struct pollfd *fds, nfds_t nfds, int timeout)
  */
 const void *io_have_fd(int fd, bool *listener);
 
-/**
- * io_set_extended_errors - enable callbacks for errors.
- * @state: true or false.
- *
- * Defaults false for compatibility.  See io_new_conn for what this changes.
- */
-void io_set_extended_errors(bool state);
-bool io_get_extended_errors(void);
-
 #endif /* CCAN_IO_H */

ccan/ccan/io/poll.c

@@ -270,12 +270,10 @@ static void accept_conn(struct io_listener *l)
 {
 	int fd = accept(l->fd.fd, NULL, NULL);
 
-	if (fd < 0) {
-		/* If they've enabled it, this is how we tell them */
-		if (io_get_extended_errors())
-			l->init(NULL, l->arg);
+	/* FIXME: What to do here? */
+	if (fd < 0)
 		return;
-	}
+
 	io_new_conn(l->ctx, fd, l->init, l->arg);
 }
 

ccan/ccan/io/test/run-43-io_plan_in_started.c

@@ -18,17 +18,9 @@ static struct io_plan *init_in_conn(struct io_conn *conn, char *buf)
 	return io_read(conn, buf, 2, in_conn_done, NULL);
 }
 
-/* Every second time we say we're exhausted */
 static int do_nothing(int fd, struct io_plan_arg *arg)
 {
-	static bool read_once;
-
-	read_once = !read_once;
-	if (read_once)
-		return 1;
-
-	errno = EAGAIN;
-	return -1;
+	return 1;
 }
 
 static struct io_plan *dummy_write(struct io_conn *conn,

ccan/ccan/likely/likely.c

@@ -29,8 +29,8 @@ static bool trace_eq(const struct trace *t1, const struct trace *t2)
 }
 
 /* struct thash */
-HTABLE_DEFINE_NODUPS_TYPE(struct trace, (const struct trace *), hash_trace, trace_eq,
-			  thash);
+HTABLE_DEFINE_TYPE(struct trace, (const struct trace *), hash_trace, trace_eq,
+		   thash);
 
 static struct thash htable
 = { HTABLE_INITIALIZER(htable.raw, thash_hash, NULL) };

ccan/ccan/lqueue/LICENSE

@@ -1 +0,0 @@
-../../licenses/BSD-MIT

ccan/ccan/lqueue/_info

@@ -1,57 +0,0 @@
-#include "config.h"
-#include <stdio.h>
-#include <string.h>
-
-/**
- * lqueue - Simple, singly-linked-list queue implementation
- *
- * This code provides a simple implementation of the Queue abstract
- * data type in terms of a singly linked (circular) list.
- *
- * License: BSD-MIT
- * Author: David Gibson <david@gibson.dropbear.id.au>
- *
- * Example:
- *	#include <ccan/lqueue/lqueue.h>
- *
- *	struct arg {
- *		const char *arg;
- *		struct lqueue_link ql;
- *	};
- *
- *	int main(int argc, char *argv[])
- *	{
- *		int i;
- *		struct arg *a;
- *		LQUEUE(struct arg, ql) argq = LQUEUE_INIT;
- *
- *		for (i = 0; i < argc; i++) {
- *			a = malloc(sizeof(*a));
- *			a->arg = argv[i];
- *			lqueue_enqueue(&argq, a);
- *		}
- *
- *		printf("Command line arguments in order:\n");
- *
- *		while (!lqueue_empty(&argq)) {
- *			a = lqueue_dequeue(&argq);
- *			printf("Argument: %s\n", a->arg);
- *			free(a);
- *		}
- *
- *		return 0;
- *	}
- */
-int main(int argc, char *argv[])
-{
-	/* Expect exactly one argument */
-	if (argc != 2)
-		return 1;
-
-	if (strcmp(argv[1], "depends") == 0) {
-		printf("ccan/tcon\n");
-		return 0;
-	}
-
-	return 1;
-}

ccan/ccan/lqueue/lqueue.h

@@ -1,238 +0,0 @@
-/* Licensed under BSD-MIT - see LICENSE file for details */
-#ifndef CCAN_LQUEUE_H
-#define CCAN_LQUEUE_H
-
-#include <stdbool.h>
-#include <stdio.h>
-#include <assert.h>
-
-#include <ccan/tcon/tcon.h>
-
-/**
- * struct lqueue_link - a queue link
- * @next: next entry, or front of queue, if this is the back
- *
- * This is used as a link within a queue entry.
- *
- * Example:
- *	struct waiter {
- *		char *name;
- *		struct lqueue_link ql;
- *	};
- */
-struct lqueue_link {
-	struct lqueue_link *next;
-};
-
-/**
- * struct lqueue_ - a queue (internal type)
- * @b: the back of the queue (NULL if empty)
- */
-struct lqueue_ {
-	struct lqueue_link *back;
-};
-
-/**
- * LQUEUE - declare a queue
- * @type: the type of elements in the queue
- * @link: the field containing the lqueue_link in @type
- *
- * The LQUEUE macro declares an lqueue.  It can be prepended by
- * "static" to define a static lqueue.  The queue begins in undefined
- * state, you must either initialize with LQUEUE_INIT, or call
- * lqueue_init() before using it.
- *
- * See also:
- *	lqueue_init()
- *
- * Example:
- *	struct element {
- *		int value;
- *		struct lqueue_link link;
- *	};
- *	LQUEUE(struct element, link) my_queue;
- */
-#define LQUEUE(etype, link)						\
-	TCON_WRAP(struct lqueue_,					\
-		  TCON_CONTAINER(canary, etype, link))
-
-/**
- * LQUEUE_INIT - initializer for an empty queue
- *
- * The LQUEUE_INIT macro returns a suitable initializer for a queue
- * defined with LQUEUE.
- *
- * Example:
- *	struct element {
- *		int value;
- *		struct lqueue_link link;
- *	};
- *	LQUEUE(struct element, link) my_queue = LQUEUE_INIT;
- *
- *	assert(lqueue_empty(&my_queue));
- */
-#define LQUEUE_INIT				\
-	TCON_WRAP_INIT({ NULL, })
-
-/**
- * lqueue_entry - convert an lqueue_link back into the structure containing it.
- * @q: the queue
- * @l: the lqueue_link
- *
- * Example:
- *	struct waiter {
- *		char *name;
- *		struct lqueue_link ql;
- *	} w;
- *	LQUEUE(struct waiter, ql) my_queue;
- *	assert(lqueue_entry(&my_queue, &w.ql) == &w);
- */
-#define lqueue_entry(q_, l_) tcon_container_of((q_), canary, (l_))
-
-/**
- * lqueue_init_from_back - initialize a queue with a specific back element
- * @s: the lqueue to initialize
- * @e: pointer to the back element of the new queue
- *
- * USE WITH CAUTION: This is for handling unusual cases where you have
- * a pointer to an element in a previously constructed queue but can't
- * conveniently pass around a normal struct lqueue.  Usually you
- * should use lqueue_init().
- *
- * Example:
- *	struct element {
- *		int value;
- *		struct lqueue_link link;
- *	} el;
- *	LQUEUE(struct element, link) queue1;
- *	LQUEUE(struct element, link) queue2;
- *
- *	lqueue_enqueue(&queue1, &el);
- *
- *	lqueue_init_from_back(&queue2, lqueue_back(&queue1));
- */
-#define lqueue_init_from_back(q_, e_)					\
-	(lqueue_init_(tcon_unwrap(q_), tcon_member_of((q_), canary, (e_))))
-
-/**
- * lqueue_init - initialize a queue
- * @h: the lqueue to set to an empty queue
- *
- * Example:
- *	struct element {
- *		int value;
- *		struct lqueue_link link;
- *	};
- *	LQUEUE(struct element, link) *qp = malloc(sizeof(*qp));
- *	lqueue_init(qp);
- */
-#define lqueue_init(q_) \
-	(lqueue_init_(tcon_unwrap(q_), NULL))
-static inline void lqueue_init_(struct lqueue_ *q, struct lqueue_link *back)
-{
-	q->back = back;
-}
-
-/**
- * lqueue_empty - is a queue empty?
- * @q: the queue
- *
- * If the queue is empty, returns true.
- */
-#define lqueue_empty(q_) \
-	lqueue_empty_(tcon_unwrap(q_))
-static inline bool lqueue_empty_(const struct lqueue_ *q)
-{
-	return (q->back == NULL);
-}
-
-/**
- * lqueue_front - get front entry in a queue
- * @q: the queue
- *
- * If the queue is empty, returns NULL.
- *
- * Example:
- *	struct element *f;
- *
- *	f = lqueue_front(qp);
- *	assert(lqueue_dequeue(qp) == f);
- */
-#define lqueue_front(q_) \
-	lqueue_entry((q_), lqueue_front_(tcon_unwrap(q_)))
-static inline struct lqueue_link *lqueue_front_(const struct lqueue_ *q)
-{
-	if (!q->back)
-		return NULL;
-	else
-		return q->back->next;
-}
-
-/**
- * lqueue_back - get back entry in a queue
- * @q: the queue
- *
- * If the queue is empty, returns NULL.
- *
- * Example:
- *	struct element b;
- *
- *	lqueue_enqueue(qp, &b);
- *	assert(lqueue_back(qp) == &b);
- */
-#define lqueue_back(q_) \
-	lqueue_entry((q_), lqueue_back_(tcon_unwrap(q_)))
-static inline struct lqueue_link *lqueue_back_(const struct lqueue_ *q)
-{
-	return q->back;
-}
-
-/**
- * lqueue_enqueue - add an entry to the back of a queue
- * @q: the queue to add the node to
- * @e: the item to enqueue
- *
- * The lqueue_link does not need to be initialized; it will be overwritten.
- */
-#define lqueue_enqueue(q_, e_)			\
-	lqueue_enqueue_(tcon_unwrap(q_), tcon_member_of((q_), canary, (e_)))
-static inline void lqueue_enqueue_(struct lqueue_ *q, struct lqueue_link *e)
-{
-	if (lqueue_empty_(q)) {
-		/* New entry will be both front and back of queue */
-		e->next = e;
-		q->back = e;
-	} else {
-		e->next = lqueue_front_(q);
-		q->back->next = e;
-		q->back = e;
-	}
-}
-
-/**
- * lqueue_dequeue - remove and return the entry from the front of the queue
- * @q: the queue
- *
- * Note that this leaves the returned entry's link in an undefined
- * state; it can be added to another queue, but not deleted again.
- */
-#define lqueue_dequeue(q_) \
-	lqueue_entry((q_), lqueue_dequeue_(tcon_unwrap(q_)))
-static inline struct lqueue_link *lqueue_dequeue_(struct lqueue_ *q)
-{
-	struct lqueue_link *front;
-
-	if (lqueue_empty_(q))
-		return NULL;
-
-	front = lqueue_front_(q);
-	if (front == lqueue_back_(q)) {
-		assert(front->next == front);
-		q->back = NULL;
-	} else {
-		q->back->next = front->next;
-	}
-	return front;
-}
-
-#endif /* CCAN_LQUEUE_H */

ccan/ccan/lqueue/test/run.c

@@ -1,69 +0,0 @@
-#include "config.h"
-
-#include <ccan/lqueue/lqueue.h>
-#include <ccan/tap/tap.h>
-
-struct waiter {
-	const char *name;
-	struct lqueue_link ql;
-};
-
-int main(void)
-{
-	LQUEUE(struct waiter, ql) q = LQUEUE_INIT;
-	struct waiter a = { "Alice" };
-	struct waiter b = { "Bob" };
-	struct waiter c = { "Carol" };
-	struct waiter *waiter;
-
-	/* This is how many tests you plan to run */
-	plan_tests(25);
-
-	ok1(lqueue_empty(&q));
-	ok1(lqueue_front(&q) == NULL);
-	ok1(lqueue_back(&q) == NULL);
-
-	lqueue_enqueue(&q, &a);
-
-	ok1(!lqueue_empty(&q));
-	ok1(lqueue_front(&q) == &a);
-	ok1(lqueue_back(&q) == &a);
-
-	lqueue_enqueue(&q, &b);
-
-	ok1(!lqueue_empty(&q));
-	ok1(lqueue_front(&q) == &a);
-	ok1(lqueue_back(&q) == &b);
-
-	lqueue_enqueue(&q, &c);
-
-	ok1(!lqueue_empty(&q));
-	ok1(lqueue_front(&q) == &a);
-	ok1(lqueue_back(&q) == &c);
-
-	waiter = lqueue_dequeue(&q);
-	ok1(waiter == &a);
-
-	ok1(!lqueue_empty(&q));
-	ok1(lqueue_front(&q) == &b);
-	ok1(lqueue_back(&q) == &c);
-
-	waiter = lqueue_dequeue(&q);
-	ok1(waiter == &b);
-
-	ok1(!lqueue_empty(&q));
-	ok1(lqueue_front(&q) == &c);
-	ok1(lqueue_back(&q) == &c);
-
-	waiter = lqueue_dequeue(&q);
-	ok1(waiter == &c);
-
-	ok1(lqueue_empty(&q));
-	ok1(lqueue_front(&q) == NULL);
-	ok1(lqueue_back(&q) == NULL);
-
-	ok1(lqueue_dequeue(&q) == NULL);
-
-	/* This exits depending on whether all tests passed */
-	return exit_status();
-}

ccan/ccan/mem/mem.h

@@ -104,7 +104,7 @@ void *memcchr(void const *data, int c, size_t data_len);
 PURE_FUNCTION
 static inline bool memeq(const void *a, size_t al, const void *b, size_t bl)
 {
-	return al == bl && (al == 0 || !memcmp(a, b, bl));
+	return al == bl && !memcmp(a, b, bl);
 }
 
 /**

ccan/ccan/membuf/_info

@@ -26,16 +26,13 @@
  *
  *	membuf_init(&charbuf, malloc(10), 10, membuf_realloc);
  *
- *	for (int i = 1; i < argc; i++) {
- *		size_t len = strlen(argv[i]);
- *		memcpy(membuf_add(&charbuf, len), argv[i], len);
- *	}
+ *	for (int i = 1; i < argc; i++)
+ *		strcpy(membuf_add(&charbuf, strlen(argv[i])), argv[i]);
  *
  *	// This is dumb, we could do all at once, but shows technique.
  *	while (membuf_num_elems(&charbuf) > 0)
  *		printf("%c", *(char *)membuf_consume(&charbuf, 1));
  *	printf("\n");
- *	free(membuf_cleanup(&charbuf));
  *	return 0;
  * }
  */

ccan/ccan/opt/helpers.c

@@ -138,11 +138,10 @@ char *opt_set_floatval(const char *arg, float *f)
 	return NULL;
 }
 
-bool opt_show_floatval(char *buf, size_t len, const float *f)
+void opt_show_floatval(char buf[OPT_SHOW_LEN], const float *f)
 {
 	double d = *f;
-	opt_show_doubleval(buf, len, &d);
-	return true;
+	opt_show_doubleval(buf, &d);
 }
 
 char *opt_set_doubleval(const char *arg, double *d)
@@ -161,10 +160,9 @@ char *opt_set_doubleval(const char *arg, double *d)
 	return NULL;
 }
 
-bool opt_show_doubleval(char *buf, size_t len, const double *d)
+void opt_show_doubleval(char buf[OPT_SHOW_LEN], const double *d)
 {
-	snprintf(buf, len, "%f", *d);
-	return true;
+	snprintf(buf, OPT_SHOW_LEN, "%f", *d);
 }
 
 char *opt_inc_intval(int *i)
@@ -198,60 +196,52 @@ char *opt_usage_and_exit(const char *extra)
 	exit(0);
 }
 
-bool opt_show_bool(char *buf, size_t len, const bool *b)
+void opt_show_bool(char buf[OPT_SHOW_LEN], const bool *b)
 {
-	strncpy(buf, *b ? "true" : "false", len);
-	return true;
+	strncpy(buf, *b ? "true" : "false", OPT_SHOW_LEN);
 }
 
-bool opt_show_invbool(char *buf, size_t len, const bool *b)
+void opt_show_invbool(char buf[OPT_SHOW_LEN], const bool *b)
 {
-	strncpy(buf, *b ? "false" : "true", len);
-	return true;
+	strncpy(buf, *b ? "false" : "true", OPT_SHOW_LEN);
 }
 
-bool opt_show_charp(char *buf, size_t len, char *const *p)
+void opt_show_charp(char buf[OPT_SHOW_LEN], char *const *p)
 {
-	if (*p) {
-		size_t plen = strlen(*p);
-		if (len < 2)
-			return false;
+	if (*p){
+		size_t len = strlen(*p);
 		buf[0] = '"';
-		if (plen > len - 2)
-			plen = len - 2;
-		strncpy(buf+1, *p, plen);
-		buf[1+plen] = '"';
-		if (plen < len - 2)
-			buf[2+plen] = '\0';
-		return true;
-	} else {
-		return false;
+		if (len > OPT_SHOW_LEN - 2)
+			len = OPT_SHOW_LEN - 2;
+		strncpy(buf+1, *p, len);
+		buf[1+len] = '"';
+		if (len < OPT_SHOW_LEN - 2)
+			buf[2+len] = '\0';
+	}
+	else {
+		strncpy(buf, "(nil)", OPT_SHOW_LEN);
 	}
 }
 
 /* Show an integer value, various forms. */
-bool opt_show_intval(char *buf, size_t len, const int *i)
+void opt_show_intval(char buf[OPT_SHOW_LEN], const int *i)
 {
-	snprintf(buf, len, "%i", *i);
-	return true;
+	snprintf(buf, OPT_SHOW_LEN, "%i", *i);
 }
 
-bool opt_show_uintval(char *buf, size_t len, const unsigned int *ui)
+void opt_show_uintval(char buf[OPT_SHOW_LEN], const unsigned int *ui)
 {
-	snprintf(buf, len, "%u", *ui);
-	return true;
+	snprintf(buf, OPT_SHOW_LEN, "%u", *ui);
 }
 
-bool opt_show_longval(char *buf, size_t len, const long *l)
+void opt_show_longval(char buf[OPT_SHOW_LEN], const long *l)
 {
-	snprintf(buf, len, "%li", *l);
-	return true;
+	snprintf(buf, OPT_SHOW_LEN, "%li", *l);
 }
 
-bool opt_show_ulongval(char *buf, size_t len, const unsigned long *ul)
+void opt_show_ulongval(char buf[OPT_SHOW_LEN], const unsigned long *ul)
 {
-	snprintf(buf, len, "%lu", *ul);
-	return true;
+	snprintf(buf, OPT_SHOW_LEN, "%lu", *ul);
 }
 
 /* a helper function that multiplies out an argument's kMGTPE suffix in the
@@ -457,14 +447,14 @@ char * opt_set_uintval_si(const char *arg, unsigned int *u)
   are separate but essentially identical functions for signed and unsigned
   values, so that unsigned values greater than LLONG_MAX get suffixes.
  */
-static void show_llong_with_suffix(char *buf, size_t len, long long ll,
-				   const long long base)
+static void show_llong_with_suffix(char buf[OPT_SHOW_LEN], long long ll,
+				    const long long base)
 {
 	const char *suffixes = "kMGTPE";
 	int i;
 	if (ll == 0){
 		/*zero is special because everything divides it (you'd get "0E")*/
-		snprintf(buf, len, "0");
+		snprintf(buf, OPT_SHOW_LEN, "0");
 		return;
 	}
 	for (i = 0; i < strlen(suffixes); i++){
@@ -474,20 +464,19 @@ static void show_llong_with_suffix(char *buf, size_t len, long long ll,
 		ll = tmp;
 	}
 	if (i == 0)
-		snprintf(buf, len, "%"PRId64, (int64_t)ll);
+		snprintf(buf, OPT_SHOW_LEN, "%"PRId64, (int64_t)ll);
 	else
-		snprintf(buf, len, "%"PRId64"%c", (int64_t)ll, suffixes[i - 1]);
+		snprintf(buf, OPT_SHOW_LEN, "%"PRId64"%c", (int64_t)ll, suffixes[i - 1]);
 }
 
-static void show_ullong_with_suffix(char *buf, size_t len,
-				    unsigned long long ull,
+static void show_ullong_with_suffix(char buf[OPT_SHOW_LEN], unsigned long long ull,
 				    const unsigned base)
 {
 	const char *suffixes = "kMGTPE";
 	int i;
 	if (ull == 0){
 		/*zero is special because everything divides it (you'd get "0E")*/
-		snprintf(buf, len, "0");
+		snprintf(buf, OPT_SHOW_LEN, "0");
 		return;
 	}
 	for (i = 0; i < strlen(suffixes); i++){
@@ -497,84 +486,72 @@ static void show_ullong_with_suffix(char *buf, size_t len,
 		ull = tmp;
 	}
 	if (i == 0)
-		snprintf(buf, len, "%"PRIu64, (uint64_t)ull);
+		snprintf(buf, OPT_SHOW_LEN, "%"PRIu64, (uint64_t)ull);
 	else
-		snprintf(buf, len, "%"PRIu64"%c", (uint64_t)ull, suffixes[i - 1]);
+		snprintf(buf, OPT_SHOW_LEN, "%"PRIu64"%c", (uint64_t)ull, suffixes[i - 1]);
 }
 
 /* _bi, signed */
-bool opt_show_intval_bi(char *buf, size_t len, const int *x)
+void opt_show_intval_bi(char buf[OPT_SHOW_LEN], const int *x)
 {
-	show_llong_with_suffix(buf, len, *x, 1024);
-	return true;
+	show_llong_with_suffix(buf, *x, 1024);
 }
 
-bool opt_show_longval_bi(char *buf, size_t len, const long *x)
+void opt_show_longval_bi(char buf[OPT_SHOW_LEN], const long *x)
 {
-	show_llong_with_suffix(buf, len, *x, 1024);
-	return true;
+	show_llong_with_suffix(buf, *x, 1024);
 }
 
-bool opt_show_longlongval_bi(char *buf, size_t len, const long long *x)
+void opt_show_longlongval_bi(char buf[OPT_SHOW_LEN], const long long *x)
 {
-	show_llong_with_suffix(buf, len, *x, 1024);
-	return true;
+	show_llong_with_suffix(buf, *x, 1024);
 }
 
 /* _bi, unsigned */
-bool opt_show_uintval_bi(char *buf, size_t len, const unsigned int *x)
+void opt_show_uintval_bi(char buf[OPT_SHOW_LEN], const unsigned int *x)
 {
-	show_ullong_with_suffix(buf, len, (unsigned long long) *x, 1024);
-	return true;
+	show_ullong_with_suffix(buf, (unsigned long long) *x, 1024);
 }
 
-bool opt_show_ulongval_bi(char *buf, size_t len, const unsigned long *x)
+void opt_show_ulongval_bi(char buf[OPT_SHOW_LEN], const unsigned long *x)
 {
-	show_ullong_with_suffix(buf, len, (unsigned long long) *x, 1024);
-	return true;
+	show_ullong_with_suffix(buf, (unsigned long long) *x, 1024);
 }
 
-bool opt_show_ulonglongval_bi(char *buf, size_t len, const unsigned long long *x)
+void opt_show_ulonglongval_bi(char buf[OPT_SHOW_LEN], const unsigned long long *x)
 {
-	show_ullong_with_suffix(buf, len, (unsigned long long) *x, 1024);
-	return true;
+	show_ullong_with_suffix(buf, (unsigned long long) *x, 1024);
 }
 
 /* _si, signed */
-bool opt_show_intval_si(char *buf, size_t len, const int *x)
+void opt_show_intval_si(char buf[OPT_SHOW_LEN], const int *x)
 {
-	show_llong_with_suffix(buf, len, (long long) *x, 1000);
-	return true;
+	show_llong_with_suffix(buf, (long long) *x, 1000);
 }
 
-bool opt_show_longval_si(char *buf, size_t len, const long *x)
+void opt_show_longval_si(char buf[OPT_SHOW_LEN], const long *x)
 {
-	show_llong_with_suffix(buf, len, (long long) *x, 1000);
-	return true;
+	show_llong_with_suffix(buf, (long long) *x, 1000);
 }
 
-bool opt_show_longlongval_si(char *buf, size_t len, const long long *x)
+void opt_show_longlongval_si(char buf[OPT_SHOW_LEN], const long long *x)
 {
-	show_llong_with_suffix(buf, len, *x, 1000);
-	return true;
+	show_llong_with_suffix(buf, *x, 1000);
 }
 
 /* _si, unsigned */
-bool opt_show_uintval_si(char *buf, size_t len, const unsigned int *x)
+void opt_show_uintval_si(char buf[OPT_SHOW_LEN], const unsigned int *x)
 {
-	show_ullong_with_suffix(buf, len, (unsigned long long) *x, 1000);
-	return true;
+	show_ullong_with_suffix(buf, (unsigned long long) *x, 1000);
 }
 
-bool opt_show_ulongval_si(char *buf, size_t len, const unsigned long *x)
+void opt_show_ulongval_si(char buf[OPT_SHOW_LEN], const unsigned long *x)
 {
-	show_ullong_with_suffix(buf, len, (unsigned long long) *x, 1000);
-	return true;
+	show_ullong_with_suffix(buf, (unsigned long long) *x, 1000);
 }
 
-bool opt_show_ulonglongval_si(char *buf, size_t len, const unsigned long long *x)
+void opt_show_ulonglongval_si(char buf[OPT_SHOW_LEN], const unsigned long long *x)
 {
-	show_ullong_with_suffix(buf, len, (unsigned long long) *x, 1000);
-	return true;
+	show_ullong_with_suffix(buf, (unsigned long long) *x, 1000);
 }
 

ccan/ccan/opt/opt.c

@@ -34,7 +34,7 @@ static const char *next_name(const char *names, unsigned *len)
 static const char *first_opt(unsigned *i, unsigned *len)
 {
 	for (*i = 0; *i < opt_count; (*i)++) {
-		if (opt_table[*i].type & OPT_SUBTABLE)
+		if (opt_table[*i].type == OPT_SUBTABLE)
 			continue;
 		return first_name(opt_table[*i].names, len);
 	}
@@ -44,7 +44,7 @@ static const char *first_opt(unsigned *i, unsigned *len)
 static const char *next_opt(const char *p, unsigned *i, unsigned *len)
 {
 	for (; *i < opt_count; (*i)++) {
-		if (opt_table[*i].type & OPT_SUBTABLE)
+		if (opt_table[*i].type == OPT_SUBTABLE)
 			continue;
 		if (!p)
 			return first_name(opt_table[*i].names, len);
@@ -114,11 +114,10 @@ static void check_opt(const struct opt_table *entry)
 {
 	const char *p;
 	unsigned len;
-	enum opt_type type = entry->type & (OPT_USER_MIN-1);
 
-	if (type != OPT_HASARG && type != OPT_NOARG
-	    && type != (OPT_EARLY|OPT_HASARG)
-	    && type != (OPT_EARLY|OPT_NOARG))
+	if (entry->type != OPT_HASARG && entry->type != OPT_NOARG
+	    && entry->type != (OPT_EARLY|OPT_HASARG)
+	    && entry->type != (OPT_EARLY|OPT_NOARG))
 		failmsg("Option %s: unknown entry type %u",
 			entry->names, entry->type);
 
@@ -162,7 +161,7 @@ static void add_opt(const struct opt_table *entry)
 void _opt_register(const char *names, enum opt_type type,
 		   char *(*cb)(void *arg),
 		   char *(*cb_arg)(const char *optarg, void *arg),
-		   bool (*show)(char *buf, size_t len, const void *arg),
+		   void (*show)(char buf[OPT_SHOW_LEN], const void *arg),
 		   const void *arg, const char *desc)
 {
 	struct opt_table opt;
@@ -182,7 +181,7 @@ bool opt_unregister(const char *names)
 	int found = -1, i;
 
 	for (i = 0; i < opt_count; i++) {
-		if (opt_table[i].type & OPT_SUBTABLE)
+		if (opt_table[i].type == OPT_SUBTABLE)
 			continue;
 		if (strcmp(opt_table[i].names, names) == 0)
 			found = i;
@@ -204,7 +203,7 @@ void opt_register_table(const struct opt_table entry[], const char *desc)
 		add_opt(&heading);
 	}
 	for (i = 0; entry[i].type != OPT_END; i++) {
-		if (entry[i].type & OPT_SUBTABLE)
+		if (entry[i].type == OPT_SUBTABLE)
 			opt_register_table(subtable_of(&entry[i]),
 					   entry[i].desc);
 		else {