Fix dockerfile for arm32/64

.dockerignore

@@ -1,3 +0,0 @@
-Dockerfile
-contrib/docker/Dockerfile.*
-target

.editorconfig

@@ -16,3 +16,6 @@ indent_size = 8
 indent_style = space
 indent_size = 4
 
+[.travis.yml]
+indent_style = space
+indent_size = 2

.gitattributes

@@ -7,14 +7,8 @@ configure text eol=lf
 # The following files are generated and should not be shown in the
 # diffs by default on Github.
 doc/lightning*.7 linguist-generated=true
-db_*_sqlgen.c linguist-generated=true
-statements_gettextgen.po linguist-generated=true
+wallet/db_*_sqlgen.c linguist-generated=true
+wallet/statements_gettextgen.po linguist-generated=true
 *_wiregen.? linguist-generated=true
 *_printgen.? linguist-generated=true
 
-# The following are marked as binary and generated since they can be
-# easily overwritten and don't need deep review.
-cln-grpc/proto/node.proto -text -diff linguist-generated=true
-cln-grpc/src/convert.rs -text -diff linguist-generated=true
-cln-rpc/src/model.rs -text -diff linguist-generated=true
-contrib/pyln-testing/pyln/testing/node_pb2.py -text -diff linguist-generated=true

.github/CODEOWNERS

@@ -4,16 +4,17 @@
 # also can optionally require approval from a code owner before the
 # author can merge a pull request in the repository.
 
-cln-grpc/               @cdecker
-cln-rpc/                @cdecker
-plugins/src/            @cdecker
-plugins/grpc-plugin/    @cdecker
-contrib/reprobuild/     @cdecker
-contrib/msggen/         @cdecker
-contrib/pyln-client/    @cdecker
-contrib/pyln-testing/   @cdecker
-# Needed to ensure hsmd wire compatibility between releases
-hsmd/hsmd_wire.csv      @cdecker @ksedgwic @devrandom
+wallet/                 @cdecker
+*.py                    @cdecker
+
+wallet/invoices.*       @ZmnSCPxj
+plugins/multifundchannel.c @ZmnSCPxj
+doc/BACKUP.md           @ZmnSCPxj @cdecker
+
+common/param.*          @wythe
+common/json.*           @wythe
+common/json_tok.*       @wythe
+common/wallet_tx.*      @wythe
 
 # See https://help.github.com/articles/about-codeowners/ for more
 # information

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,16 +0,0 @@
-> [!IMPORTANT]
->
-> 25.09 FREEZE July 28TH: Non-bugfix PRs not ready by this date will wait for 25.12.
->
-> RC1 is scheduled on _August 11th_
->
-> The final release is scheduled for September 1st.
-
-
-## Checklist
-Before submitting the PR, ensure the following tasks are completed. If an item is not applicable to your PR, please mark it as checked:
-
-- [ ] The changelog has been updated in the relevant commit(s) according to the [guidelines](https://docs.corelightning.org/docs/coding-style-guidelines#changelog-entries-in-commit-messages).
-- [ ] Tests have been added or modified to reflect the changes.
-- [ ] Documentation has been reviewed and updated as needed.
-- [ ] Related issues have been listed and linked, including any that this PR closes.

.github/scripts/build.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+echo "Running in $(pwd)"
+export ARCH=${ARCH:-64}
+export BOLTDIR=lightning-rfc
+export CC=${COMPILER:-gcc}
+export COMPAT=${COMPAT:-1}
+export TEST_CHECK_DBSTMTS=${TEST_CHECK_DBSTMTS:-0}
+export DEVELOPER=${DEVELOPER:-1}
+export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0}
+export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
+export PYTEST_OPTS="--maxfail=5 --suppress-no-test-exit-code ${PYTEST_OPTS}"
+export PYTEST_PAR=${PYTEST_PAR:-10}
+export PYTEST_SENTRY_ALWAYS_REPORT=1
+export SLOW_MACHINE=1
+export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"}
+export TEST_DB_PROVIDER=${TEST_DB_PROVIDER:-"sqlite3"}
+export TEST_NETWORK=${NETWORK:-"regtest"}
+export TIMEOUT=900
+export VALGRIND=${VALGRIND:-0}
+export FUZZING=${FUZZING:-0}
+export LIGHTNINGD_POSTGRES_NO_VACUUM=1
+
+pip3 install --user -U \
+     -r requirements.lock
+
+timeout 60 pip3 install --user \
+     --use-feature=in-tree-build \
+     ./contrib/pyln-client \
+     ./contrib/pyln-proto \
+     ./contrib/pyln-testing
+
+# Install utilities that aren't dependencies, but make
+# running tests easier/feasible on CI (and pytest which
+# keeps breaking the rerunfailures plugin).
+pip3 install --user \
+     blinker \
+     flake8 \
+     flaky \
+     mako \
+     pytest-sentry \
+     pytest-test-groups==1.0.3 \
+     pytest-custom-exit-code==0.3.0 \
+     pytest-timeout \
+     pytest-json-report
+
+git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc
+git submodule update --init --recursive
+
+./configure CC="$CC"
+cat config.vars
+
+cat << EOF > pytest.ini
+[pytest]
+addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+markers =
+    slow_test: marks tests as slow (deselect with '-m "not slow_test"')
+EOF
+
+if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ]
+then
+    export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/
+    export MAKE_HOST="$TARGET_HOST"
+    export BUILD=x86_64-pc-linux-gnu
+    export AR="$TARGET_HOST"-ar
+    export AS="$TARGET_HOST"-as
+    export CC="$TARGET_HOST"-gcc
+    export CXX="$TARGET_HOST"-g++
+    export LD="$TARGET_HOST"-ld
+    export STRIP="$TARGET_HOST"-strip
+    export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static
+
+    wget -q https://zlib.net/zlib-1.2.11.tar.gz
+    tar xf zlib-1.2.11.tar.gz
+    cd zlib-1.2.11 || exit 1
+    ./configure --prefix="$QEMU_LD_PREFIX"
+    make
+    sudo make install
+    cd .. || exit 1
+    rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11
+
+    wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip
+    unzip -q sqlite-src-3260000.zip
+    cd sqlite-src-3260000 || exit 1
+    automake --add-missing --force-missing --copy || true
+    ./configure --disable-tcl \
+     --enable-static \
+     --disable-readline \
+     --disable-threadsafe \
+     --disable-load-extension \
+     --host="$TARGET_HOST" \
+     --prefix="$QEMU_LD_PREFIX"
+    make
+    sudo make install
+    cd .. || exit 1
+    rm sqlite-src-3260000.zip
+    rm -rf sqlite-src-3260000
+
+    wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz
+    tar xf gmp-6.1.2.tar.xz
+    cd gmp-6.1.2 || exit 1
+    ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST"
+    make
+    sudo make install
+    cd ..
+    rm gmp-6.1.2.tar.xz
+    rm -rf gmp-6.1.2
+
+    ./configure CC="$TARGET_HOST-gcc" --enable-static
+
+    make -j32 CC="$TARGET_HOST-gcc" > /dev/null
+else
+    eatmydata make -j32
+    # shellcheck disable=SC2086
+    eatmydata $TEST_CMD
+fi

.github/scripts/install-bitcoind.sh

@@ -1,31 +0,0 @@
-#!/bin/sh
-
-set -e
-
-export BITCOIN_VERSION=27.1
-export ELEMENTS_VERSION=23.2.1
-
-DIRNAME="bitcoin-${BITCOIN_VERSION}"
-EDIRNAME="elements-${ELEMENTS_VERSION}"
-FILENAME="${DIRNAME}-x86_64-linux-gnu.tar.gz"
-EFILENAME="${EDIRNAME}-x86_64-linux-gnu.tar.gz"
-
-cd /tmp/
-
-# Since we inadvertently broke `elementsd` support in the past we only
-# want to download and enable the daemon that is actually going to be
-# used when running in CI. Otherwise we could end up accidentally
-# testing against `bitcoind` but still believe that we ran against
-# `elementsd`.
-if [ "$TEST_NETWORK" = "liquid-regtest" ]; then
-    wget "https://github.com/ElementsProject/elements/releases/download/elements-${ELEMENTS_VERSION}/${EFILENAME}"
-    tar -xf "${EFILENAME}"
-    sudo mv "${EDIRNAME}"/bin/* "/usr/local/bin"
-    rm -rf "${EFILENAME}" "${EDIRNAME}"
-else
-    wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/${FILENAME}"
-    tar -xf "${FILENAME}"
-    sudo mv "${DIRNAME}"/bin/* "/usr/local/bin"
-    rm -rf "${FILENAME}" "${DIRNAME}"
-fi
-

.github/scripts/setup.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
-set -e
-export DEBIAN_FRONTEND=noninteractive
-export RUST_VERSION=stable
 
+export DEBIAN_FRONTEND=noninteractive
+export BITCOIN_VERSION=0.20.1
+export ELEMENTS_VERSION=0.18.1.8
 sudo useradd -ms /bin/bash tester
 sudo apt-get update -qq
 
@@ -20,18 +20,12 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      gcc-arm-none-eabi \
      gettext \
      git \
-     gnupg \
-     jq \
      libc6-dev-arm64-cross \
      libc6-dev-armhf-cross \
-     libev-dev \
-     libevent-dev \
-     libffi-dev \
-     libicu-dev \
+     libgmp-dev \
      libpq-dev \
      libprotobuf-c-dev \
      libsqlite3-dev \
-     libssl-dev \
      libtool \
      libxml2-utils \
      locales \
@@ -49,45 +43,26 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      software-properties-common \
      sudo \
      tcl \
-     tclsh \
      unzip \
      valgrind \
      wget \
      xsltproc \
-     systemtap-sdt-dev \
      zlib1g-dev
 
 echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
 sudo chmod 0440 /etc/sudoers.d/tester
 
-"$(dirname "$0")"/install-bitcoind.sh
-
-curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
-     -y --default-toolchain ${RUST_VERSION}
-
-# We also need a relatively recent protobuf-compiler, at least 3.12.0,
-# in order to support the experimental `optional` flag.
-
-# BUT WAIT!  Gentoo wants this to match the version from the Python protobuf,
-# which comes from the same tree.  Makes sense!
-
-# And
-#   grpcio-tools-1.69.0` requires `protobuf = ">=5.26.1,<6.0dev"`
-
-# Now, protoc changed to date-based releases, BUT Python protobuf
-# didn't, so Python protobuf 4.21.12 (in Ubuntu 23.04) corresponds to
-# protoc 21.12 (which, FYI, is packaged in Ubuntu as version 3.21.12).
-
-# In general protobuf version x.y.z corresponds to protoc version y.z
-
-# Honorable mention go to Matt Whitlock for spelunking this horror with me!
-
-PROTOC_VERSION=29.4
-PB_REL="https://github.com/protocolbuffers/protobuf/releases"
-curl -LO $PB_REL/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip
-sudo unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local/
-sudo chmod a+x /usr/local/bin/protoc
-export PROTOC=/usr/local/bin/protoc
-export PATH=$PATH:/usr/local/bin
-env
-ls -lha /usr/local/bin
+(
+    cd /tmp/ || exit 1
+    wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2
+    wget -q https://storage.googleapis.com/c-lightning-tests/elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2
+    tar -xjf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2
+    tar -xjf elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2
+    sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin
+    sudo mv elements-$ELEMENTS_VERSION/bin/* /usr/local/bin
+    rm -rf \
+       bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz \
+       bitcoin-$BITCOIN_VERSION \
+       elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 \
+       elements-$ELEMENTS_VERSION
+)

.github/scripts/sync-rpc-cmds.py

@@ -1,115 +0,0 @@
-import os
-from time import sleep
-import requests
-import re
-from enum import Enum
-
-# readme url
-URL = "https://dash.readme.com/api/v1"
-# category id for API reference
-CATEGORY_ID = "63e4e160c60b2e001dd1cc4e"
-CATEGORY_SLUG = "json-rpc-apis"
-
-
-class Action(Enum):
-    ADD = 'add'
-    UPDATE = 'update'
-    DELETE = 'delete'
-
-
-def getListOfRPCDocs(headers):
-    response = requests.get(f"{URL}/categories/{CATEGORY_SLUG}/docs", headers=headers)
-    if response.status_code == 200:
-        return response.json()
-    else:
-        return []
-
-
-def publishDoc(action, title, body, order, headers):
-    payload = {
-        "title": title,
-        "type": "basic",
-        "body": body,
-        "category": CATEGORY_ID,
-        "hidden": False,
-        "order": order,
-    }
-    # title == slug
-    if action == Action.ADD:
-        # create doc
-        response = requests.post(URL + "/docs", json=payload, headers=headers)
-        if response.status_code != 201:
-            print(response.text)
-        else:
-            print("Created ", title)
-    elif action == Action.UPDATE:
-        # update doc
-        response = requests.put(f"{URL}/docs/{title}", json=payload, headers=headers)
-        if response.status_code != 200:
-            print(response.text)
-        else:
-            print("Updated ", title)
-    elif action == Action.DELETE:
-        # delete doc
-        response = requests.delete(f"{URL}/docs/{title}", headers=headers)
-        if response.status_code != 204:
-            print(response.text)
-        else:
-            print("Deleted ", title)
-    else:
-        print("Invalid action")
-
-
-def extract_rpc_commands(rst_content):
-    manpages_block = re.search(
-        r"\.\. block_start manpages(.*?)" r"\.\. block_end manpages",
-        rst_content,
-        re.DOTALL,
-    )
-    if manpages_block:
-        commands = re.findall(
-            r"\b([a-zA-Z0-9_-]+)" r"\s+<([^>]+)>\n", manpages_block.group(1)
-        )
-        return commands
-    return []
-
-
-def main():
-    # define headers for requests
-    headers = {
-        "accept": "application/json",
-        "content-type": "application/json",
-        "authorization": "Basic " + os.environ.get("README_API_KEY"),
-    }
-
-    # path to the rst file from where we fetch all the RPC commands
-    path_to_rst = "doc/index.rst"
-    with open(path_to_rst, "r") as file:
-        rst_content = file.read()
-
-    commands_from_local = extract_rpc_commands(rst_content)
-    commands_from_readme = getListOfRPCDocs(headers)
-
-    # Compare local and server commands list to get the list of command to add or delete
-    commands_local_title = set(command[0] for command in commands_from_local)
-    commands_readme_title = set(command['title'] for command in commands_from_readme)
-    commands_to_delete = commands_readme_title - commands_local_title
-    commands_to_add = commands_local_title - commands_readme_title
-    for name in commands_to_delete:
-        publishDoc(Action.DELETE, name, "", 0, headers)
-        sleep(3)
-
-    if commands_from_local:
-        order = 0
-        for name, file in commands_from_local:
-            with open("doc/" + file) as f:
-                body = f.read()
-                publishDoc(Action.ADD if name in commands_to_add else Action.UPDATE, name, body, order, headers)
-            order = order + 1
-            sleep(3)
-    else:
-        print("No commands found in the Manpages block.")
-
-
-if __name__ == "__main__":
-    main()

.github/workflows/bsd.yml

@@ -1,79 +1,86 @@
 name: FreeBSD Test
 
-on:
-  push:
-    branches:
-      - "master"
-  pull_request:
-  workflow_dispatch:
+on: [push, pull_request]
 
 jobs:
   testfreebsd:
-    runs-on: ubuntu-22.04
+    runs-on: macos-10.15
     name: Build and test on FreeBSD
-    timeout-minutes: 120
-    strategy:
-      fail-fast: true
-      matrix:
-        bitcoind-version: ["27.1"]
+    env:
+      DEVELOPER: 1
+      VALGRIND: 0
+      EXPERIMENTAL_FEATURES: 0
+      COMPAT: 1
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v2
     - name: Test in FreeBSD
       id: test
-      uses: vmactions/freebsd-vm@v1
+      uses: vmactions/freebsd-vm@v0.1.5
       with:
+        envs: 'DEVELOPER VALGRIND EXPERIMENTAL_FEATURES COMPAT'
         usesh: true
-        sync: rsync
-        copyback: false
         prepare: |
           pkg install -y \
-            bash \
             wget \
-            python310 \
+            py38-pip \
+            py38-sqlite3 \
             gmake \
             git \
             python \
-            postgresql16-server \
+            postgresql12-server \
             autoconf \
             automake \
             libtool \
+            gmp \
             bash \
             gettext \
             sqlite3 \
-            lowdown \
-            pkgconf \
-            jq \
-            protobuf \
             curl
 
-            python3.10 -m ensurepip
-            python3.10 -m pip install --upgrade pip
-            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
-
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2021-08-3z1
 
+          cd /tmp/ || exit 1
+          wget https://storage.googleapis.com/c-lightning-tests/bitcoin-0.20.1-x86_64-linux-gnu.tar.bz2
+          tar -xjf bitcoin-0.20.1-x86_64-linux-gnu.tar.bz2
+          sudo mv bitcoin-0.20.1/bin/* /usr/local/bin
+          rm -rf \
+            bitcoin-0.20.1-x86_64-linux-gnu.tar.gz \
+            bitcoin-0.20.1
+          
         run: |
-          PATH=/root/.local/bin:$PATH:/root/.cargo/bin; export PATH
+          PATH=/root/.local/bin:$PATH
           pip install --user -U wheel pip
-          pip3 install --user poetry
-          poetry install
+          pip install --user -U -r requirements.txt
 
-          git clone https://github.com/lightning/bolts.git ../bolts
-          # fatal: unsafe repository ('/Users/runner/work/lightning/lightning' is owned by someone else)
-          git config --global --add safe.directory `pwd`
-          for d in libsodium libwally-core gheap jsmn libbacktrace lowdown; do git config --global --add safe.directory `pwd`/external/$d; done
+          # Install utilities that aren't dependencies, but make
+          # running tests easier/feasible on CI (and pytest which
+          # keeps breaking the rerunfailures plugin).
+          pip install --user -U \
+            blinker \
+            flake8 \
+            mako \
+            mrkd \
+            pytest-sentry \
+            pytest-test-groups==1.0.3 \
+            pytest-custom-exit-code==0.3.0 \
+            pytest-json-report
+
+          git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc
           git submodule update --init --recursive
 
-          ./configure CC="$CC" --disable-valgrind
-
+          ./configure CC="$CC"
           cat config.vars
 
           cat << EOF > pytest.ini
           [pytest]
-          addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+          addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
           markers =
               slow_test: marks tests as slow (deselect with '-m "not slow_test"')
           EOF
 
           # Just run a "quick" test without memory checking
-          poetry run gmake
+          gmake
+
+          # Clean up to maximize rsync's chances of succeeding
+          gmake clean
 

.github/workflows/ci.yaml

@@ -1,653 +1,339 @@
 ---
 name: Continuous Integration
-on:
-  push:
-    branches:
-      - "master"
-  pull_request:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-env:
-  # Makes the upload-artifact work more reliably at the cost
-  # of a bit of compile time.
-  RUST_PROFILE: release
-  SLOW_MACHINE: 1
-  CI_SERVER_URL: "http://35.239.136.52:3170"
-
+on: [push, pull_request]
 jobs:
-  prebuild:
-    name: Pre-build checks
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    env:
-      BOLTDIR: bolts
-    strategy:
-      fail-fast: true
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
-
-      - name: Rebase
-        # We can't rebase if we're on master already.
-        if: github.ref != 'refs/heads/master'
-        run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-          git fetch origin ${{ github.base_ref }}
-          git rebase origin/${{ github.base_ref }}
-
-      - name: Check changelog
-        env:
-          PR_DESCRIPTION: "${{ github.event.pull_request.body || '' }}"
-          EVENT_NAME: "${{ github.event_name }}"
-          BASE_REF: "${{ github.base_ref || 'master' }}"
-        run: |
-          echo "Event Name: $EVENT_NAME"
-          echo "Base Ref: $BASE_REF"
-          echo "PR DESCRIPTION: $PR_DESCRIPTION"
-          if [ "$EVENT_NAME" = "pull_request" ]; then
-            if [[ "$PR_DESCRIPTION" != *"Changelog-None"* && \
-                  -z "$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')" && \
-                  "$(git rev-parse --abbrev-ref HEAD)" != "$BASE_REF" ]]; then
-              echo "::error::'Changelog' entry is missing in all commits, and 'Changelog-None' not specified in the PR description"
-              exit 1
-            else
-              if [[ "$PR_DESCRIPTION" == *"Changelog-None"* ]]; then
-                echo "Changelog found in PR description"
-              else
-                echo "Changelog found in Commit \"$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')\""
-              fi
-            fi
-          else
-            echo "Not a PR event."
-          fi
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          # Export and then use pip to install into the current env
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-          # We're going to check BOLT quotes, so get the latest version
-          git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
-      - name: Configure
-        run: ./configure --enable-debugbuild --enable-rust
-      - name: Check source
-        env:
-          VALGRIND: 0
-          PYTEST_OPTS: --timeout=1200
-        run: make check-source BASE_REF="origin/${{ github.base_ref }}"
-      - name: Check Generated Files have been updated
-        run: make check-gen-updated
-      - name: Check docs
-        run: make check-doc
-
-  compile:
-    name: Compile CLN ${{ matrix.cfg }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    needs:
-      - prebuild
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - CFG: compile-gcc
-            VALGRIND: 1
-            COMPILER: gcc
-          - CFG: compile-gcc-O3
-            VALGRIND: 1
-            COMPILER: gcc
-            COPTFLAGS_VAR: COPTFLAGS="-O3 -Werror"
-          # While we're at it let's try to compile with clang
-          - CFG: compile-clang
-            VALGRIND: 1
-            COMPILER: clang
-          - CFG: compile-clang-sanitizers
-            COMPILER: clang
-            ASAN: 1
-            UBSAN: 1
-            VALGRIND: 0
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-
-      - name: Build
-        env:
-          COMPILER: ${{ matrix.COMPILER }}
-          ASAN: ${{ matrix.ASAN }}
-          UBSAN: ${{ matrix.UBSAN }}
-          VALGRIND: ${{ matrix.VALGRIND }}
-          COMPAT: 1
-          CFG: ${{ matrix.CFG }}
-        run: |
-          set -e
-          pip3 install --user pip wheel poetry
-          poetry self add poetry-plugin-export
-          poetry export -o requirements.txt --with dev --without-hashes
-          python3 -m pip install -r requirements.txt
-          ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }}
-
-          make -j $(nproc) testpack.tar.bz2
-
-          # Rename now so we don't clash
-          mv testpack.tar.bz2 cln-${CFG}.tar.bz2
-      - name: Check rust packages
-        run: cargo test --all
-      - uses: actions/upload-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-          path: cln-${{ matrix.CFG }}.tar.bz2
-
-  check-units:
-    # The unit test checks are not in the critical path (not dependent
-    # on the integration tests), so run them with `valgrind`
-    name: Run unit tests
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    env:
-      BOLTDIR: bolts
-    needs:
-      - compile
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - CFG: compile-gcc
-            VALGRIND: 1
-          - CFG: compile-clang-sanitizers
-            VALGRIND: 0
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          sudo apt-get update -qq
-          sudo apt-get install -y -qq lowdown
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          # Export and then use pip to install into the current env
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-          # We're going to check BOLT quotes, so get the latest version
-          git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-
-      - name: Check
-        run: |
-          tar -xaf cln-${{ matrix.CFG }}.tar.bz2
-          eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }}
-
-  check-fuzz:
-    name: Run fuzz regression tests
-    runs-on: ubuntu-22.04
-    needs:
-      - prebuild
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          # Export and then use pip to install into the current env
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-
-      - name: Build
-        run: |
-          ./configure --enable-debugbuild --enable-fuzzing --enable-address-sanitizer --enable-ub-sanitizer --disable-valgrind CC=clang
-          make -j $(nproc) check-fuzz
-
-  integration:
-    name: Test CLN ${{ matrix.name }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    env:
-      RUST_PROFILE: release  # Has to match the one in the compile step
-      PYTEST_OPTS: --timeout=1200
-    needs:
-      - compile
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - NAME: gcc
-            CFG: compile-gcc
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: gcc
-            TEST_NETWORK: regtest
-          # While we're at it let's try to compile with clang
-          - NAME: clang
-            CFG: compile-clang
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: clang
-            TEST_NETWORK: regtest
-          # And of course we want to test postgres too
-          - NAME: postgres
-            CFG: compile-gcc
-            COMPILER: gcc
-            TEST_DB_PROVIDER: postgres
-            TEST_NETWORK: regtest
-          # And don't forget about elements (like cdecker did when
-          # reworking the CI...)
-          - NAME: liquid
-            CFG: compile-gcc
-            COMPILER: gcc
-            TEST_NETWORK: liquid-regtest
-            TEST_DB_PROVIDER: sqlite3
-          # And dual funding!
-          - NAME: dual-fund
-            CFG: compile-gcc
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: gcc
-            TEST_NETWORK: regtest
-            EXPERIMENTAL_DUAL_FUND: 1
-          # And splicing!
-          - NAME: splicing
-            CFG: compile-gcc
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: gcc
-            TEST_NETWORK: regtest
-            EXPERIMENTAL_SPLICING: 1
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          pip3 install --user pip wheel poetry
-          poetry install
-
-      - name: Install bitcoind
-        env:
-          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
-        run: .github/scripts/install-bitcoind.sh
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-
-      - name: Unpack pre-built CLN
-        env:
-          CFG: ${{ matrix.CFG }}
-        run: |
-          tar -xaf cln-${CFG}.tar.bz2
-
-      - name: Switch network
-        if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }}
-        run: |
-          # Loading the network from config.vars rather than the envvar is a terrible idea...
-          sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars
-          cat config.vars
-
-      - name: Test
-        env:
-          COMPILER: ${{ matrix.COMPILER }}
-          EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }}
-          EXPERIMENTAL_SPLICING: ${{ matrix.EXPERIMENTAL_SPLICING }}
-          COMPAT: 1
-          CFG: ${{ matrix.CFG }}
-          SLOW_MACHINE: 1
-          PYTEST_PAR: 10
-          TEST_DEBUG: 1
-          TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
-          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
-          LIGHTNINGD_POSTGRES_NO_VACUUM: 1
-        run: |
-          env
-          cat config.vars
-          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
-
-  integration-valgrind:
-    name: Valgrind Test CLN ${{ matrix.name }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    env:
-      RUST_PROFILE: release  # Has to match the one in the compile step
-      CFG: compile-gcc
-      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800
-    needs:
-      - compile
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - NAME: Valgrind (01/10)
-            PYTEST_OPTS: --test-group=1  --test-group-count=10
-          - NAME: Valgrind (02/10)
-            PYTEST_OPTS: --test-group=2  --test-group-count=10
-          - NAME: Valgrind (03/10)
-            PYTEST_OPTS: --test-group=3  --test-group-count=10
-          - NAME: Valgrind (04/10)
-            PYTEST_OPTS: --test-group=4  --test-group-count=10
-          - NAME: Valgrind (05/10)
-            PYTEST_OPTS: --test-group=5  --test-group-count=10
-          - NAME: Valgrind (06/10)
-            PYTEST_OPTS: --test-group=6  --test-group-count=10
-          - NAME: Valgrind (07/10)
-            PYTEST_OPTS: --test-group=7  --test-group-count=10
-          - NAME: Valgrind (08/10)
-            PYTEST_OPTS: --test-group=8  --test-group-count=10
-          - NAME: Valgrind (09/10)
-            PYTEST_OPTS: --test-group=9  --test-group-count=10
-          - NAME: Valgrind (10/10)
-            PYTEST_OPTS: --test-group=10 --test-group-count=10
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          sudo apt-get update -qq
-          sudo apt-get install -yyq valgrind
-          pip3 install --user pip wheel poetry
-          poetry install
-
-      - name: Install bitcoind
-        run: .github/scripts/install-bitcoind.sh
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-compile-gcc.tar.bz2
-
-      - name: Unpack build
-        run: tar -xvjf cln-compile-gcc.tar.bz2
-
-      - name: Test
-        env:
-          SLOW_MACHINE: 1
-          TEST_DEBUG: 1
-        run: |
-          VALGRIND=1 poetry run eatmydata pytest tests/ -vvv -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
-
-  integration-sanitizers:
-    name: Sanitizers Test CLN
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    env:
-      RUST_PROFILE: release
-      SLOW_MACHINE: 1
-      TEST_DEBUG: 1
-      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800
-    needs:
-      - compile
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - NAME: ASan/UBSan (01/10)
-            PYTEST_OPTS: --test-group=1  --test-group-count=10
-          - NAME: ASan/UBSan (02/10)
-            PYTEST_OPTS: --test-group=2  --test-group-count=10 -n 1
-          - NAME: ASan/UBSan (03/10)
-            PYTEST_OPTS: --test-group=3  --test-group-count=10
-          - NAME: ASan/UBSan (04/10)
-            PYTEST_OPTS: --test-group=4  --test-group-count=10
-          - NAME: ASan/UBSan (05/10)
-            PYTEST_OPTS: --test-group=5  --test-group-count=10
-          - NAME: ASan/UBSan (06/10)
-            PYTEST_OPTS: --test-group=6  --test-group-count=10
-          - NAME: ASan/UBSan (07/10)
-            PYTEST_OPTS: --test-group=7  --test-group-count=10
-          - NAME: ASan/UBSan (08/10)
-            PYTEST_OPTS: --test-group=8  --test-group-count=10
-          - NAME: ASan/UBSan (09/10)
-            PYTEST_OPTS: --test-group=9  --test-group-count=10
-          - NAME: ASan/UBSan (10/10)
-            PYTEST_OPTS: --test-group=10 --test-group-count=10
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          set -e
-          pip3 install --user wheel poetry
-          poetry install --with dev --no-root
-
-      - name: Install bitcoind
-        run: .github/scripts/install-bitcoind.sh
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-compile-clang-sanitizers.tar.bz2
-
-      - name: Unpack build
-        run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2
-
-      - name: Test
-        run: |
-          poetry run eatmydata pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
-
-  update-docs-examples:
-    name: Update examples in doc schemas (disabled temporarily!)
-    if: false
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
+  smoke-test:
+    name: Smoke Test ${{ matrix.cfg }}
+    runs-on: ubuntu-20.04
     env:
+      DEVELOPER: 1
       VALGRIND: 0
-      GENERATE_EXAMPLES: 1
-      PYTEST_OPTS: --timeout=1200
-      TEST_NETWORK: regtest
-    needs:
-      - compile
+      EXPERIMENTAL_FEATURES: 0
+      COMPAT: 1
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - CFG: "make"
+            TEST_CMD: "make"
+          - CFG: "make-O3-check"
+            TEST_CMD: "make check-source check-units installcheck check-gen-updated"
+            COPTFLAGS: "-O3"
+          - CFG: "make-32-bit-nodev-check"
+            ARCH: 32
+            TEST_CMD: "make check-source check-units installcheck"
+            DEVELOPER: 0
+          - CFG: "make-EXPERIMENTAL-check"
+            TEST_CMD: "make check-source check-units installcheck check-gen-updated"
+            EXPERIMENTAL_FEATURES: 1
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+        uses: actions/checkout@v2.0.0
+
+      - name: Fetch tags for auto versioning
+        run: git fetch --prune --unshallow --tags -f
+
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
         with:
-          python-version: '3.10'
+          python-version: 3.6
+
       - name: Install dependencies
         run: |
           bash -x .github/scripts/setup.sh
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-      - name: Install bitcoind
-        env:
-          TEST_NETWORK: regtest
-        run: .github/scripts/install-bitcoind.sh
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-compile-gcc.tar.bz2
-      - name: Unpack pre-built CLN
-        run: |
-          tar -xaf cln-compile-gcc.tar.bz2
-      - name: Test
-        run: |
-          eatmydata make -j $(nproc) check-doc-examples
 
-  min-btc-support:
-    name: Test minimum supported BTC v${{ matrix.MIN_BTC_VERSION }} with ${{ matrix.NAME }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
+      - name: Build
+        env:
+          VALGRIND: ${{ matrix.VALGRIND }}
+          DEVELOPER: ${{ matrix.DEVELOPER }}
+          EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }}
+          COMPILER: ${{ matrix.COMPILER }}
+          ARCH: ${{ matrix.ARCH }}
+          COMPAT: ${{ matrix.COMPAT }}
+          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
+          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
+          COPTFLAGS: ${{ matrix.COPTFLAGS }}
+          NETWORK: ${{ matrix.NETWORK }}
+          TEST_CMD: ${{ matrix.TEST_CMD }}
+          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
+          TEST_GROUP: ${{ matrix.TEST_GROUP }}
+        run: |
+          bash -x .github/scripts/build.sh
+
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Junit Report ${{ github.run_number }}.${{ matrix.cfg }}
+          path: report.*
+          if-no-files-found: ignore
+
+  check-dock:
+    name: Check c-lightning doc
+    runs-on: ubuntu-20.04
     env:
-      RUST_PROFILE: release  # Has to match the one in the compile step
-      PYTEST_OPTS: --timeout=1200
-    needs:
-      - compile
+      DEVELOPER: 1
+      VALGRIND: 0
+      EXPERIMENTAL_FEATURES: 0
+      COMPAT: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2.0.0
+
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+
+      - name: Install dependencies
+        run: bash -x .github/scripts/setup.sh
+
+      - name: Check Doc
+        run: |
+          pip install mako
+          ./configure
+          make check-doc
+
+  proto-test:
+    name: Protocol Test Config
+    runs-on: ubuntu-20.04
+    needs: [smoke-test]
+    env:
+      DEVELOPER: 1
+      EXPERIMENTAL_FEATURES: 1
+      COMPAT: 0
+      PYTEST_PAR: 2
+      TEST_CMD: "make check-protos"
+      TEST_GROUP: 1
+      TEST_GROUP_COUNT: 1
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - {compiler: clang, db: sqlite3}
+          - {compiler: gcc, db: postgres}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2.0.0
+
+      - name: Setup Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+
+      - name: Install dependencies
+        run: |
+          bash -x .github/scripts/setup.sh
+
+      - name: Build
+        env:
+          ARCH: ${{ matrix.arch }}
+          COMPILER: ${{ matrix.compiler }}
+          DB: ${{ matrix.db }}
+          NETWORK: ${{ matrix.network }}
+          TARGET_HOST: ${{ matrix.TARGET_HOST }}
+          VALGRIND: ${{ matrix.valgrind }}
+        run: |
+          bash -x .github/scripts/build.sh
+
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Junit Report ${{ github.run_number }}.{{ matrix.cfg }}
+          path: report.*
+
+  normal-test:
+    name: Normal Test Config ${{ matrix.cfg }}
+    runs-on: ubuntu-20.04
+    needs: [smoke-test]
+    env:
+      DEVELOPER: 1
+      VALGRIND: 0
+      EXPERIMENTAL_FEATURES: 0
+      COMPAT: 1
     strategy:
       fail-fast: false
       matrix:
         include:
-          - NAME: clang
-            CFG: compile-clang
-            TEST_DB_PROVIDER: sqlite3
+          # All of the following will just run `make pytest`
+          - CFG: "clang-fuzzing"
             COMPILER: clang
-            TEST_NETWORK: regtest
-            MIN_BTC_VERSION: '25.0'
+            FUZZING: 1
+          - CFG: "check-dbstmts"
+            COMPILER: gcc
+            TEST_CHECK_DBSTMTS: 1
+          - CFG: "non-DEVELOPER-non-COMPAT-1"
+            DEVELOPER: 0
+            COMPAT: 0
+            TEST_GROUP: 1
+            TEST_GROUP_COUNT: 2
+          - CFG: "non-DEVELOPER-non-COMPAT-2"
+            DEVELOPER: 0
+            COMPAT: 0
+            TEST_GROUP: 2
+            TEST_GROUP_COUNT: 2
+          - CFG: "DUAL_FUND"
+            EXPERIMENTAL_DUAL_FUND: 1
+            DEVELOPER: 1
+            COMPAT: 0
+          # Various other configurations
+          - CFG: "Elements"
+            NETWORK: liquid-regtest
+          - CFG: "PostgreSQL"
+            DB: postgres
+            PYTEST_PAR: 2
+
+          # The cross-compiled versions
+          - CFG: "cross-arm32"
+            ARCH: arm32v7
+            TARGET_HOST: arm-linux-gnueabihf
+          - CFG: "cross-arm64"
+            ARCH: arm64v8
+            TARGET_HOST: aarch64-linux-gnu
+
+          # The experimental feature test
+          - CFG: "EXPERIMENTAL"
+            EXPERIMENTAL_FEATURES: 1
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2.0.0
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
         with:
-          python-version: '3.10'
+          python-version: 3.6
 
       - name: Install dependencies
         run: |
-          pip3 install --user pip wheel poetry
-          poetry install
+          bash -x .github/scripts/setup.sh
 
-      - name: Download Bitcoin Core
-        run: wget "https://bitcoincore.org/bin/bitcoin-core-${{ matrix.MIN_BTC_VERSION }}/bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz"
-
-      - name: Extract Bitcoin Core
-        run: tar -xf "bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz"
-
-      - name: Move Bitcoin Core Binaries
-        run: sudo mv bitcoin-${{ matrix.MIN_BTC_VERSION }}/bin/* /usr/local/bin/
-
-      - name: Clean Up Downloaded Bitcoin
-        run: rm -rf "bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz" "bitcoin-${{ matrix.MIN_BTC_VERSION }}"
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-
-      - name: Unpack pre-built CLN
-        env:
-          CFG: ${{ matrix.CFG }}
-        run: |
-          tar -xaf cln-${CFG}.tar.bz2
-
-      - name: Test
+      - name: Build
         env:
+          VALGRIND: ${{ matrix.VALGRIND }}
+          DEVELOPER: ${{ matrix.DEVELOPER }}
+          EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }}
+          EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }}
           COMPILER: ${{ matrix.COMPILER }}
-          COMPAT: 1
-          CFG: ${{ matrix.CFG }}
-          SLOW_MACHINE: 1
-          PYTEST_PAR: 10
-          TEST_DEBUG: 1
-          TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
-          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
-          LIGHTNINGD_POSTGRES_NO_VACUUM: 1
+          ARCH: ${{ matrix.ARCH }}
+          COMPAT: ${{ matrix.COMPAT }}
+          FUZZING: ${{ matrix.FUZZING }}
+          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
+          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
+          NETWORK: ${{ matrix.NETWORK }}
+          TEST_CMD: ${{ matrix.TEST_CMD }}
+          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
+          TEST_GROUP: ${{ matrix.TEST_GROUP }}
+          TEST_DB_PROVIDER: ${{ matrix.DB }}
         run: |
-          env
-          cat config.vars
-          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
+          bash -x .github/scripts/build.sh
 
-  check-flake:
-    name: Check Nix Flake
-    runs-on: ubuntu-22.04
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Junit Report ${{ github.run_number }}.${{ matrix.cfg }}
+          path: report.*
+
+  valgrind-test:
+    name: Valgrind Test Config ${{ matrix.cfg }}
+    runs-on: ubuntu-20.04
+    needs: [smoke-test]
+    env:
+      DEVELOPER: 1
+      EXPERIMENTAL_FEATURES: 0
+      COMPAT: 1
+      TEST_GROUP_COUNT: 10
+      PYTEST_PAR: 3
+      LABEL: "Valgrind-test"
     strategy:
       fail-fast: true
+      matrix:
+        include:
+          - CFG: "valgrind-1"
+            VALGRIND: 1
+            TEST_GROUP: 1
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-2"
+            VALGRIND: 1
+            TEST_GROUP: 2
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-3"
+            VALGRIND: 1
+            TEST_GROUP: 3
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-4"
+            VALGRIND: 1
+            TEST_GROUP: 4
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-5"
+            VALGRIND: 1
+            TEST_GROUP: 5
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-6"
+            VALGRIND: 1
+            TEST_GROUP: 6
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-7"
+            VALGRIND: 1
+            TEST_GROUP: 7
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-8"
+            VALGRIND: 1
+            TEST_GROUP: 8
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-9"
+            VALGRIND: 1
+            TEST_GROUP: 9
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
+          - CFG: "valgrind-10"
+            VALGRIND: 1
+            TEST_GROUP: 10
+            TEST_GROUP_COUNT: 10
+            PYTEST_PAR: 3
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: 'recursive'
-      - name: Check Nix flake inputs
-        uses: DeterminateSystems/flake-checker-action@v8
-      - name: Install Nix
-        uses: cachix/install-nix-action@V27
-        with:
-          nix_path: nixpkgs=channel:nixos-24.05
-      - name: Check flake
-        run: nix flake check .?submodules=1#
+        uses: actions/checkout@v2.0.0
 
-  gather:
-    # A dummy task that depends on the full matrix of tests, and
-    # signals successful completion. Used for the PR status to pass
-    # before merging.  Needs to run even if they failed!
-    name: CI completion
-    runs-on: ubuntu-22.04
-    needs:
-      - integration
-      - check-units
-      - integration-valgrind
-      - integration-sanitizers
-      - min-btc-support
-      - check-flake
-    if: ${{ always() }}
-    steps:
-      - name: Complete
-        env:
-          JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC FLAKE"
-          INTEGRATION: ${{ needs.integration.result }}
-          CHECK_UNITS: ${{ needs['check-units'].result }}
-          VALGRIND: ${{ needs['integration-valgrind'].result }}
-          SANITIZERS: ${{ needs['integration-sanitizers'].result }}
-          DOCS: ${{ needs['update-docs-examples'].result }}
-          BTC: ${{ needs['min-btc-support'].result }}
-          FLAKE: ${{ needs['check-flake'].result }}
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+
+      - name: Install dependencies
         run: |
-          failed=""
-          for name in $JOB_NAMES; do
-            result="${!name}"
-            echo "$name: $result"
-            if [[ "$result" != "success" ]]; then
-              failed="yes"
-            fi
-          done
-          if [[ "$failed" == "yes" ]]; then
-            echo "One or more required jobs failed"
-            exit 1
-          fi
+          bash -x .github/scripts/setup.sh
+
+      - name: Build
+        env:
+          VALGRIND: ${{ matrix.VALGRIND }}
+          DEVELOPER: ${{ matrix.DEVELOPER }}
+          EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }}
+          EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }}
+          COMPILER: ${{ matrix.COMPILER }}
+          ARCH: ${{ matrix.ARCH }}
+          COMPAT: ${{ matrix.COMPAT }}
+          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
+          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
+          NETWORK: ${{ matrix.NETWORK }}
+          TEST_CMD: ${{ matrix.TEST_CMD }}
+          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
+          TEST_GROUP: ${{ matrix.TEST_GROUP }}
+        run: |
+          bash -x .github/scripts/build.sh
+
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Junit Report ${{ github.run_number }}.${{ matrix.cfg }}
+          path: report.*

.github/workflows/crate-io.yml

@@ -1,30 +0,0 @@
----
-name: "Release Rust 🦀"
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - "master"
-
-jobs:
-  release_rust:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@stable
-      - uses: arduino/setup-protoc@v3
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './cln-rpc'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './plugins'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './cln-grpc'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true

.github/workflows/docker-release.yml

@@ -1,118 +0,0 @@
-name: Build and push multi-platform docker images
-
-on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      version:
-        description: 'Release version'
-        required: true
-
-      repository-name:
-        description: 'Docker repository name'
-        default: 'elementsproject'
-        required: false
-
-      platforms-to-build:
-        description: 'List of platforms to build'
-        default: 'linux/amd64,linux/arm64,linux/arm/v7'
-        required: false
-  
-      push-latest:
-        description: 'Push the latest tag also?'
-        default: 'false'
-        required: false
-
-jobs:
-  build:
-    runs-on: ubuntu-22.04
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-      with:
-        ref: ${{ github.ref }}  # Ensures the branch triggering the workflow is checked out
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-
-    - name: Setup Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
-    - name: Log in to Docker Hub
-      uses: docker/login-action@v3
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
-    - name: Set up values
-      id: set-values
-      run: |
-        if [[ "${{ github.event.inputs.version }}" != "" ]]; then
-          echo "Input version provided"
-          VERSION=${{ github.event.inputs.version }}
-        elif [[ ${{ github.ref_type }} == "tag" ]]; then
-          echo "This is a tag event"
-          VERSION=${{ github.ref_name }}
-        else
-          echo "No version provided and no tag found."
-          exit 1
-        fi
-        echo "VERSION=$VERSION" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.repository-name }}" != "" ]]; then
-          REPONAME=${{ github.event.inputs.repository-name }}
-        else
-          REPONAME="elementsproject"
-        fi
-        echo "REPONAME=$REPONAME" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.platforms-to-build }}" != "" ]]; then
-          PLATFORMS=${{ github.event.inputs.platforms-to-build }}
-        else
-          PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
-        fi
-        echo "PLATFORMS=$PLATFORMS" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.push-latest }}" == "true" ]] || 
-          ([[ "${{ github.ref_type }}" == "tag" ]] && [[ ! "$VERSION" =~ rc ]]); then
-          echo "Latest true"
-          PUSHLATEST="true"
-        else
-          echo "Latest false"
-          PUSHLATEST="false"
-        fi
-        echo "PUSHLATEST=$PUSHLATEST" >> $GITHUB_ENV
-
-        TAGS="$REPONAME/lightningd:$VERSION"
-        if [[ "$PUSHLATEST" == "true" ]]; then
-          TAGS="$TAGS,$REPONAME/lightningd:latest"
-        fi
-        echo "TAGS=$TAGS" >> $GITHUB_ENV
-
-    - name: Print GitHub Ref Values
-      run: |
-        echo "GITHUB REF TYPE: ${{ github.ref_type }}"
-        echo "GITHUB REF NAME: ${{ github.ref_name }}"
-        echo "EVENT INPUT VERSION: ${{ github.event.inputs.version }}"
-        echo "EVENT INPUT REPO: ${{ github.event.inputs.repository-name }}"
-        echo "EVENT INPUT PLATFORMS: ${{ github.event.inputs.platforms-to-build }}"
-        echo "EVENT INPUT PUSH LATEST: ${{ github.event.inputs.push-latest }}"
-        echo "ENV VERSION: ${{ env.VERSION }}"
-        echo "ENV REPO NAME: ${{ env.REPONAME }}"
-        echo "ENV PLATFORMS: ${{ env.PLATFORMS }}"
-        echo "ENV PUSH LATEST: ${{ env.PUSHLATEST }}"
-        echo "ENV TAGS: ${{ env.TAGS }}"
-
-    - name: Build and push Docker image
-      uses: docker/build-push-action@v5
-      with:
-        context: .
-        file: ./Dockerfile
-        push: true
-        platforms: ${{ env.PLATFORMS }}
-        tags: ${{ env.TAGS }}

.github/workflows/docker.yml

@@ -1,34 +0,0 @@
-name: Docker packaging
-on:
-  push:
-    tags:
-      - 'basedon-*'
-
-jobs:
-  main:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Environment variables
-        run: env
-      -
-        name: Create images
-        env:
-          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
-          DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }}
-          DOCKERHUB_REPO: ${{ vars.DOCKERHUB_REPO }}
-        shell: bash
-        run: |
-          LATEST_TAG=${GITHUB_REF#refs/tags/}
-          LATEST_TAG=${LATEST_TAG:8} #trim "basedon-" from tag
-          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USER" --password-stdin
-          docker buildx create --use
-          DOCKER_BUILDX_OPTS="--platform linux/amd64,linux/arm64,linux/arm/v7 --push"
-          docker buildx build $DOCKER_BUILDX_OPTS -t $DOCKERHUB_REPO:$LATEST_TAG .

.github/workflows/macos.yaml

@@ -1,62 +1,102 @@
 ---
 name: Mac OS pytest
-on:
-  pull_request:
+on: [pull_request]
 jobs:
   smoke-test:
     name: Smoke Test macOS
-    runs-on: macos-14
-    timeout-minutes: 120
+    runs-on: macos-latest
+    env:
+      DEVELOPER: 1
+      VALGRIND: 0
+      EXPERIMENTAL_FEATURES: 1
+      COMPAT: 0
     strategy:
       fail-fast: true
-      matrix:
-        bitcoind-version: ["27.1"]
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2.0.0
 
-      - name: Download Bitcoin ${{ matrix.bitcoind-version }} & install binaries
-        run: |
-          export BITCOIND_VERSION=${{ matrix.bitcoind-version }}
-          export TARGET_ARCH="arm64-apple-darwin"
+      - name: Fetch tags for auto versioning
+        run: git fetch --prune --unshallow --tags
 
-          wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
-          tar -xzf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
-          sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin
-          rm -rf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz bitcoin-${BITCOIND_VERSION}
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
 
       - name: Install dependencies
         run: |
-          export PATH="/usr/local/opt:/Users/runner/.local/bin:/opt/homebrew/bin/python3.10/bin:$PATH"
+          export BITCOIN_VERSION=0.20.1
+          brew install wget python autoconf automake libtool python3 gmp gnu-sed gettext libsodium
 
-          brew install gnu-sed python@3.10 autoconf automake libtool protobuf
-          python3.10 -m pip install -U --user poetry==1.8.0 wheel pip mako
-          python3.10 -m poetry install
+          (
+            cd /tmp/
+            wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-osx64.tar.gz -O bitcoin.tar.gz
+            tar -xvzf bitcoin.tar.gz
+            sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin
+          )
 
-      - name: Build and install CLN
+          pip install --upgrade mako pip
+          ln -s /usr/local/Cellar/gettext/0.20.1/bin/xgettext /usr/local/opt
+          export PATH="/usr/local/opt:$PATH"
+
+      - name: Build
+        env:
+          VALGRIND: ${{ matrix.VALGRIND }}
+          DEVELOPER: ${{ matrix.DEVELOPER }}
+          EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }}
+          COMPILER: ${{ matrix.COMPILER }}
+          COMPAT: ${{ matrix.COMPAT }}
+          PYTEST_PAR: ${{ matrix.PYTEST_PAR }}
+          PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }}
+          NO_PYTHON: ${{ matrix.NO_PYTHON }}
+          COPTFLAGS: ${{ matrix.COPTFLAGS }}
+          NETWORK: ${{ matrix.NETWORK }}
+          TEST_CMD: ${{ matrix.TEST_CMD }}
+          TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
+          TEST_GROUP: ${{ matrix.TEST_GROUP }}
         run: |
-          export CPATH=/opt/homebrew/include
-          export LIBRARY_PATH=/opt/homebrew/lib
+          export PATH="/usr/local/opt:/Users/runner/.local/bin:$PATH"
+          export LDFLAGS="-L/usr/local/opt/sqlite/lib"
+          export CPPFLAGS="-I/usr/local/opt/sqlite/include"
 
-          python3.10 -m poetry run ./configure --disable-valgrind --disable-compat
-          python3.10 -m poetry run make
+          pip3 install --user -U \
+            -r requirements.lock
 
-      - name: Start bitcoind in regtest mode
-        run: |
-          bitcoind -regtest -daemon
-          sleep 5
+          pip3 install --user --no-index \
+            --use-feature=in-tree-build \
+            ./contrib/pyln-spec/bolt7 \
+            ./contrib/pyln-client \
+            ./contrib/pyln-proto \
+            ./contrib/pyln-testing
 
-      - name: Generate initial block
-        run: |
-          bitcoin-cli -regtest createwallet default_wallet
-          bitcoin-cli -regtest generatetoaddress 1 $(bitcoin-cli -regtest getnewaddress)
-          sleep 2
+          # Install utilities that aren't dependencies, but make
+          # running tests easier/feasible on CI (and pytest which
+          # keeps breaking the rerunfailures plugin).
+          pip3 install --user -U \
+            blinker \
+            flake8 \
+            flaky \
+            mako \
+            pytest-sentry \
+            pytest-test-groups==1.0.3 \
+            pytest-custom-exit-code==0.3.0 \
+            pytest-json-report
 
-      - name: Start CLN in regtest mode
-        run: |
-          lightningd/lightningd --network=regtest --log-file=/tmp/l1.log --daemon
-          sleep 5
+          cat << EOF > pytest.ini
+          [pytest]
+          addopts=-p no:logging --color=yes --timeout=600 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+          markers =
+              slow_test: marks tests as slow (deselect with '-m "not slow_test"')
+          EOF
 
-      - name: Verify CLN is running
-        run: |
-          cli/lightning-cli --regtest getinfo
+          ./configure
+          make
+
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: Junit Report ${{ github.run_number }}.${{ matrix.cfg }}
+          path: report.*
+          if-no-files-found: ignore

.github/workflows/pypi.yml

@@ -1,23 +1,14 @@
+---
 name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
-
 on:
+  # Only deploy if we're the result of a PR being merged
   push:
-    tags:
-      - 'v[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      dist-location:
-        description: 'Distribution location (test/prod)'
-        default: 'test'
-        required: false
-
+    branches:
+      - master
 jobs:
   deploy:
     name: Build and publish ${{ matrix.package }} 🐍
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: true
       matrix:
@@ -28,74 +19,54 @@ jobs:
             WORKDIR: contrib/pyln-testing
           - PACKAGE: pyln-proto
             WORKDIR: contrib/pyln-proto
+          - PACKAGE: pyn-bolt1
+            WORKDIR: contrib/pyln-spec/bolt1/
+          - PACKAGE: pyn-bolt2
+            WORKDIR: contrib/pyln-spec/bolt2/
+          - PACKAGE: pyn-bolt4
+            WORKDIR: contrib/pyln-spec/bolt4/
+          - PACKAGE: pyn-bolt7
+            WORKDIR: contrib/pyln-spec/bolt7/
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+    - uses: actions/checkout@master
       with:
         # Need to fetch entire history in order to locate the version tag
         fetch-depth: 0
 
-    - name: Check version tag
-      run: >-
-        git describe --tags --always --dirty=-modded --abbrev=7
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
 
-    - name: Setup Version
+    - name: Install pypa/build
+      run: >-
+        python -m pip install build --user
+
+    - name: Build a binary wheel and a source tarball
       env:
         WORKDIR: ${{ matrix.WORKDIR }}
-      run: |
-        echo "VERSION=$(git describe --tags --abbrev=0).post$(git describe --tags --abbrev=1 | awk -F "-" '{print $2}')" >> $GITHUB_ENV
-
-    - name: Set up values
-      id: set-values
-      run: |
-        if [[ "${{ github.event.inputs.dist-location }}" != "" ]]; then
-          DISTLOCATION=${{ github.event.inputs.dist-location }}
-        elif [[ "${{ github.ref_type }}" == "tag" ]] && [[ ! "${{ github.ref_name }}" =~ rc ]]; then
-          DISTLOCATION="prod"
-        else
-          DISTLOCATION="test"
-        fi
-        echo "DISTLOCATION=$DISTLOCATION" >> $GITHUB_OUTPUT
-        echo "EVENT DISTLOCATION: ${{ github.event.inputs.dist-location }}"
-        echo "DISTRIBUTION LOCATION: $DISTLOCATION"
-  
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.10'
-
-    - name: Install Poetry
-      run: |
-        curl -sSL https://install.python-poetry.org | python3 -
-        echo "$HOME/.local/bin" >> $GITHUB_PATH
-        echo "PATH=$HOME/.local/bin:$PATH"
+      run: >-
+        cd ${{ env.WORKDIR}} &&
+        python -m build --sdist --wheel --outdir dist/ .
 
     - name: Publish distribution 📦 to Test PyPI
-      if: github.repository == 'ElementsProject/lightning' && steps.set-values.outputs.DISTLOCATION == 'test'
+      if: github.repository == 'ElementsProject/lightning'
+      uses: pypa/gh-action-pypi-publish@master
       env:
-        POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.TEST_PYPI_API_TOKEN }}
         WORKDIR: ${{ matrix.WORKDIR }}
-      run: |
-        echo "POETRY VERSION TEST: $(poetry --version)"
-        echo "Pyln VERSION: $VERSION"
-        cd ${{ env.WORKDIR }}
-        python3 -m pip config set global.timeout 150
-        poetry config repositories.testpypi https://test.pypi.org/legacy/
-        make upgrade-version NEW_VERSION=$VERSION
-        poetry build  --no-interaction
-        poetry publish --repository testpypi --no-interaction --skip-existing
+      with:
+        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        repository_url: https://test.pypi.org/legacy/
+        packages_dir: "${{ env.WORKDIR}}/dist"
+        skip_existing: true
 
     - name: Publish distribution 📦 to PyPI
-      if: github.repository == 'ElementsProject/lightning' && steps.set-values.outputs.DISTLOCATION == 'prod'
+      if: startsWith(github.ref, 'refs/tags') && github.repository == 'ElementsProject/lightning'
+      uses: pypa/gh-action-pypi-publish@master
       env:
-        POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
         WORKDIR: ${{ matrix.WORKDIR }}
-      run: |
-        echo "POETRY VERSION PUBLISH: $(poetry --version)"
-        cd ${{ env.WORKDIR }}
-        export VERSION=$(git describe --tags --abbrev=0)
-        echo "Pyln VERSION: $VERSION"
-        make upgrade-version NEW_VERSION=$VERSION
-        python3 -m pip config set global.timeout 150
-        poetry build  --no-interaction
-        poetry publish --no-interaction
+      with:
+        password: ${{ secrets.PYPI_API_TOKEN }}
+        packages_dir: "${{ env.WORKDIR}}/dist"
+        # We should never have a conflict here, the version tags are unique
+        skip_existing: false

.github/workflows/rdme-docs-sync.yml

@@ -1,51 +0,0 @@
-name: ReadMe Sync
-
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'doc/**'
-  workflow_dispatch:
-
-jobs:
-  rdme-docs-sync:  
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Check out repo 📚
-        uses: actions/checkout@v4
-
-      - name: Sync doc/getting-started/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/getting-started --key=${{ env.README_API_KEY }} --version=1
-            
-      - name: Sync doc/beginners-guide/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/beginners-guide --key=${{ env.README_API_KEY }} --version=1
-      
-      - name: Sync doc/node-operators-guide/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/node-operators-guide --key=${{ env.README_API_KEY }} --version=1
-      
-      - name: Sync doc/developers-guide/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/developers-guide --key=${{ env.README_API_KEY }} --version=1
-      
-      - name: Sync doc/contributing-to-core-lightning/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/contribute-to-core-lightning --key=${{ env.README_API_KEY }} --version=1

.github/workflows/readme-rpc-sync.yml

@@ -1,43 +0,0 @@
-name: ReadMe RPC Sync
-
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'doc/schemas/*.json'
-      - 'doc/*.1.md'
-      - 'doc/*.5.md'
-      - 'doc/*.8.md'
-      - 'doc/lightningd-*.7.md'
-      - 'doc/reckless.7.md'
-  workflow_dispatch:
-
-jobs:
-  rdme-rpc-sync:
-    runs-on: ubuntu-22.04
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
-
-    - name: Install python modules
-      run: |
-        python -m pip install requests mako grpcio-tools
-
-    - name: Install dependencies
-      run: bash -x .github/scripts/setup.sh
-
-    - name: Build (including rpc .md files)
-      run: |
-        ./configure --enable-debugbuild
-        make -j $(nproc)
-
-    - name: Set environment variable and run
-      env:
-        README_API_KEY: ${{ secrets.README_API_KEY }}
-      run: python .github/scripts/sync-rpc-cmds.py

.github/workflows/release.yml

@@ -1,191 +0,0 @@
----
-# https://docs.corelightning.org/docs/release-checklist
-name: "Release 🚀"
-on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      version:
-        description: 'Release version'
-        required: true
-      create_release:
-        description: Create a draft release
-        default: no
-        type: choice
-        options:
-          - yes
-          - no
-
-jobs:
-  check:
-    name: Check
-    outputs:
-      version: ${{ steps.capture.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.ref }}
-          fetch-depth: 0
-          fetch-tags: true
-
-      - name: Determine version
-        run: |
-          if [[ "${{ github.event.inputs.version }}" != "" ]]; then
-            VERSION="${{ github.event.inputs.version }}"
-          elif [ "${{ github.ref_type }}" == "tag" ]; then
-            VERSION="${{ github.ref_name }}"
-          else
-            echo "No release version provided and no tag found."
-            exit 1
-          fi
-          echo "VERSION=$VERSION" >> "$GITHUB_ENV"
-          echo "Determined version: $VERSION"
-
-      - name: Validate release
-        run: tools/check-release.sh --version=${VERSION}
-
-      - name: Catpure version output
-        id: capture
-        run: echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
-
-  releases:
-    name: Releases
-    needs: check
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false    # Let each build finish.
-      matrix:
-        target:
-          - 'bin-Fedora'
-          - 'bin-Ubuntu-focal'
-          - 'bin-Ubuntu-jammy'
-          - 'bin-Ubuntu-noble'
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-tags: true
-
-      # tools/build-release.sh requires lowdown
-      - name: Prepare base environment
-        run: |
-          sudo apt-get install -y lowdown
-          ./configure
-
-      - name: Build environment setup
-        run: |
-          distribution=$(echo ${{ matrix.target }} | cut -d'-' -f3)
-          echo "Building base image for ${distribution}"
-          sudo docker run --rm -v $(pwd):/build ubuntu:${distribution} bash -c "\
-              apt-get update && \
-              apt-get install -y debootstrap && \
-              debootstrap ${distribution} /build/${distribution}"
-          sudo tar -C ${distribution} -c . | docker import - ${distribution}
-
-          # Build Docker image
-          docker build -t cl-repro-${distribution} - < contrib/reprobuild/Dockerfile.${distribution}
-        if: contains(matrix.target, 'Ubuntu')
-
-      - name: Build release
-        run: tools/build-release.sh ${{ matrix.target }}
-
-      - name: Upload target artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          path: release/
-          name: ${{ matrix.target }}
-          if-no-files-found: error
-
-  artifact:
-    name: Construct release artifact
-    needs:
-      - check
-      - releases
-    env:
-      version: ${{ needs.check.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Merge artifacts
-        uses: actions/upload-artifact/merge@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          pattern: bin-*
-          delete-merged: true
-
-  release:
-    name: Sign and prepare release draft
-    needs:
-      - check
-      - artifact
-    env:
-      version: ${{ needs.check.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-tags: true
-
-      - name: Download artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          path: release/
-
-      - name: Import GPG keys
-        id: gpg
-        uses: crazy-max/ghaction-import-gpg@v6
-        with:
-          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.GPG_PASSPHRASE }}
-          trust_level: 5
-
-      - name: Set default GPG key
-        run: echo "default-key ${{ steps.gpg.outputs.keyid }}" >> ~/.gnupg/gpg.conf
-
-      - name: Sign release
-        run: |
-          sudo apt-get install -y lowdown
-          ./configure
-          tools/build-release.sh --without-zip sign
-          mv release/SHA256SUMS.asc${{ steps.gpg.outputs.keyid  }} release/SHA256SUMS.asc
-
-      - name: Upload signed artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          overwrite: true
-          path: release/
-
-      - name: Determine release data
-        id: release_data
-        run: |
-          VERSION=${{ env.version }}
-          CHANGELOG_VERSION=${VERSION#v}
-          echo "CHANGELOG_VERSION=$CHANGELOG_VERSION"
-          echo "changelog_version=$CHANGELOG_VERSION" >> "$GITHUB_OUTPUT"
-
-          CHANGELOG_TITLE=$(grep "## \[${CHANGELOG_VERSION}\]" CHANGELOG.md)
-          echo "CHANGELOG_TITLE=$CHANGELOG_TITLE"
-          echo "changelog_title=$CHANGELOG_TITLE" >> "$GITHUB_OUTPUT"
-
-          RELEASE_TITLE=$(echo $CHANGELOG_TITLE | cut -d'"' -f2)
-          echo "RELEASE_TITLE=$RELEASE_TITLE"
-          echo "release_title=$RELEASE_TITLE" >> "$GITHUB_OUTPUT"
-
-      - name: Prepare release draft
-        if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.create_release == 'yes')
-        uses: softprops/action-gh-release@v2
-        with:
-          name: "${{ env.version }} ${{ steps.release_data.outputs.release_title }}"
-          tag_name: ${{ env.version }}
-          draft: true
-          prerelease: contains(env.version, "-rc")
-          files: release/*
-          fail_on_unmatched_files: true

.github/workflows/repro.yml

@@ -1,113 +0,0 @@
----
-# https://docs.corelightning.org/docs/repro
-name: Repro Build Nightly
-on:
-  # 05:00 Berlin, 03:00 UTC, 23:00 New York, 20:00 Los Angeles
-  schedule:
-    - cron: "0 3 * * *"
-  workflow_dispatch:
-
-jobs:
-  ubuntu:
-    name: "Ubuntu repro build: ${{ matrix.version }}"
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: false    # Let each build finish.
-      matrix:
-        version: ['focal', 'jammy', 'noble']
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-
-      - name: Build environment setup - ${{ matrix.version }}
-        run: |
-          echo "Building base image for ${{ matrix.version }}"
-          echo "STEP=Build environment setup" >> "$GITHUB_ENV"
-          sudo docker run --rm -v $(pwd):/build ubuntu:${{ matrix.version }} bash -c "\
-            apt-get update && \
-            apt-get install -y debootstrap && \
-            debootstrap ${{ matrix.version }} /build/${{ matrix.version }}"
-          sudo tar -C ${{ matrix.version }} -c . | docker import - ${{ matrix.version }}
-
-      - name: Builder image setup - ${{ matrix.version }}
-        run: |
-          echo "STEP=Builder image setup" >> "$GITHUB_ENV"
-          docker build -t cl-repro-${{ matrix.version }} - < contrib/reprobuild/Dockerfile.${{ matrix.version }}
-
-      - name: Build reproducible image and store Git state - ${{ matrix.version }}
-        run: |
-          echo "STEP=Build reproducible image and store Git state" >> "$GITHUB_ENV"
-
-          # Create release directory.
-          mkdir $GITHUB_WORKSPACE/release
-
-          # Perform the repro build.
-          docker run --name cl-build -v $GITHUB_WORKSPACE:/repo -e FORCE_MTIME=$(date +%F) -t cl-repro-${{ matrix.version }}
-
-          # Commit the image in order to inspect the build later.
-          docker commit cl-build cl-repro
-
-          # Inspect the version.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "make version > /repo/release/version.txt"
-
-          # Inspect the Git tree state.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "\
-              git --no-pager status > /repo/release/git.log && \
-              git --no-pager diff >> /repo/release/git.log"
-
-          # Change permissions on the release files for access by the runner environment.
-          sudo chown -R runner $GITHUB_WORKSPACE/release
-
-      - name: Assert clean version - ${{ matrix.version }}
-        run: |
-          echo "STEP=Assert clean version" >> "$GITHUB_ENV"
-          echo 'Version:'
-          cat release/version.txt
-          echo -e
-
-          releasefile=$(ls release/clightning-*)
-          echo 'Release file:'
-          ls -al release/clightning-*
-          echo -e
-          if [ -n "$(cat release/version.txt | sed -n '/-modded/p')" ] || \
-             [ -n "$(echo $releasefile | sed -n '/-modded/p')" ]; then
-            echo "Git Status and Diff:"
-            cat release/git.log
-            echo -e
-            echo 'Error: release modded / dirty tree.'
-            exit 1
-          else
-            echo 'Success! Clean release.'
-          fi
-
-      - name: Upload release artifact - ${{ matrix.version }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: release-${{ matrix.version }}
-          path: release
-          retention-days: 3 # Automatically delete after 3 days
-
-      - name: Send email on failure
-        if: ${{ failure() }}
-        uses: dawidd6/action-send-mail@v3
-        with:
-          server_address: smtp.gmail.com
-          server_port: 587
-          username: ${{ secrets.EMAIL_USERNAME }}
-          password: ${{ secrets.EMAIL_PASSWORD }}
-          from: ${{ secrets.EMAIL_USERNAME }}
-          to: ${{ vars.DISTRIBUTION_LIST }}
-          subject: "CI Failure: Step ${{ env.STEP }} failed for distro ${{ matrix.version }}"
-          convert_markdown: true
-          html_body: |
-            <html>
-              <body>
-                <p>GitHub Workflow ${{ github.workflow }} Failed! For more details, click on the action below.</p>
-                <strong>Failure Details:</strong><br/>
-                <strong>Event: </strong>${{ github.event_name }}<br/>
-                <strong>Job: </strong>${{ github.job }}<br/>
-                <strong>Distro: </strong>${{ matrix.version }}<br/>
-                <strong>Step: </strong>${{ env.STEP }}<br/>
-                <strong>Action: </strong><a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}</a><br/>
-              </body>
-            </html>

.gitignore

@@ -9,8 +9,6 @@
 *.dSYM
 *.rej
 *.pyc
-*.tmp
-libccan.a
 .cppcheck-suppress
 .mypy_cache
 TAGS
@@ -34,8 +32,6 @@ monkeytype.sqlite3
 !*/test/run-*.c
 */test/exp-run-*
 !*/test/exp-run-*.c
-*/*/test/run-*
-!*/*/test/run-*.c
 test/test_protocol
 test/test_sphinx
 tests/.pytest.restart
@@ -53,52 +49,17 @@ contrib/pyln-*/dist/
 contrib/pyln-*/pyln_*.egg-info/
 contrib/pyln-*/.eggs/
 contrib/pyln-*/pyln/*/__version__.py
+release/
 tests/plugins/test_selfdisable_after_getmanifest
 
 # Ignore generated files
 devtools/features
-doc/schemas/sql.json
-doc/*.7.md
-doc/*.[1578]
-doc/reckless*.[1578]
+doc/lightning*.[1578]
 *_sqlgen.[ch]
 *_wiregen.[ch]
 *_printgen.[ch]
 *_gettextgen.po
-tests/node_pb2.py
-tests/node_pb2_grpc.py
-tests/primitives_pb2.py
-tests/primitives_pb2_grpc.py
-tests/autogenerate-examples-status.log
-tests/autogenerate-examples.json
-tests/autogenerate-examples-repeat.log
 
 # Ignore unrelated stuff
 .DS_Store
 .gdb_history
-.python-version
-compile_commands.json
-
-# Rust targets
-target
-plugins/cln-grpc
-plugins/clnrest
-
-# Build directories
-bionic/
-focal/
-jammy/
-noble/
-release/
-.vscode/
-.cache/
-
-# Ignore release verification Sha256Sums
-SHA256SUMS-*
-doc/.doc_version
-autogenerate-examples-status.log
-
-# Ignore nix outputs
-result
-result-[0-9]*
-tests/plugins/channeld_fakenet

.gitlab-ci.yml

@@ -10,7 +10,7 @@ build:
       curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN"
       -X POST
       --data '{"state": "pending", "description": "Gitlab-CI is building the commit", "context": "gitlab-ci"}'
-      https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_COMMIT_SHA || true
+      https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_BUILD_REF || true
   script:
     - make
     - make -j 12 check
@@ -27,7 +27,7 @@ update-status-fail:
   when: on_failure
   script:
     - >-
-      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "failure", "description": "Gitlab-CI build failed, please contact @cdecker for details about build #$CI_JOB_ID.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_COMMIT_SHA || true
+      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "failure", "description": "Gitlab-CI build failed, please contact @cdecker for details about build #$CI_BUILD_ID.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_BUILD_REF || true
 
 update-status-success:
   image: tutum/curl
@@ -35,4 +35,4 @@ update-status-success:
   when: on_success
   script:
     - >-
-      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "success", "description": "Gitlab-CI build succeeded.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_COMMIT_SHA || true
+      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "success", "description": "Gitlab-CI build succeeded.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_BUILD_REF || true

.gitmodules

@@ -14,7 +14,6 @@
 [submodule "external/gheap"]
 	path = external/gheap
 	url = https://github.com/valyala/gheap
-[submodule "external/lowdown"]
-	path = external/lowdown
-	url = https://github.com/kristapsdz/lowdown.git
-	ignore = dirty
+[submodule "external/lnprototest"]
+	path = external/lnprototest
+	url = https://github.com/rustyrussell/lnprototest.git

.pre-commit-config.yaml

@@ -1,13 +0,0 @@
-repos:
-- repo: https://github.com/astral-sh/ruff-pre-commit
-  # Ruff version.
-  rev: v0.8.0
-  hooks:
-    # Run the linter.
-    - id: ruff
-      args: [ --diff ]
-      exclude: "contrib/pyln-grpc-proto/pyln/grpc/(primitives|node)_pb2(|_grpc).py"
-    # Run the formatter.
-    - id: ruff-format
-      args: [ --diff ]
-      exclude: "contrib/pyln-grpc-proto/pyln/grpc/(primitives|node)_pb2(|_grpc).py"

.version

@@ -1 +0,0 @@
-25.05

Cargo.toml

@@ -1,13 +0,0 @@
-[profile.release]
-strip = "debuginfo"
-
-[workspace]
-resolver = "2"
-members = [
-    "cln-rpc",
-    "cln-grpc",
-    "plugins",
-    "plugins/grpc-plugin",
-    "plugins/rest-plugin",
-    "plugins/lsps-plugin",
-]

Dockerfile

@@ -1,35 +1,11 @@
-# This Dockerfile is used by buildx to build ARM64, AMD64, and ARM32 Docker images from an AMD64 host.
-# To speed up the build process, we are cross-compiling rather than relying on QEMU.
-# There are four main stages:
-# * downloader: Downloads specific binaries needed for core lightning for each architecture.
-# * builder: Cross-compiles for each architecture.
-# * builder-python: Builds Python dependencies for wss-proxy with QEMU.
-# * final: Creates the runtime image.
-
-ARG DEFAULT_TARGETPLATFORM="linux/amd64"
-ARG BASE_DISTRO="debian:bookworm-slim"
-
-FROM --platform=$BUILDPLATFORM ${BASE_DISTRO} AS base-downloader
-RUN set -ex \
-	&& apt-get update \
-	&& apt-get install -qq --no-install-recommends ca-certificates dirmngr wget qemu-user-static binfmt-support
-
-FROM base-downloader AS base-downloader-linux-amd64
-ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu
-ENV DESCHASHPLUGIN_ARCH=linux-amd64
-ENV DESCHASHPLUGIN_HASH=deadc00c68fac80b2718d92f69bf06acd8fff646228d497bbb76a4f0a12ca217
-
-FROM base-downloader AS base-downloader-linux-arm64
-ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu
-ENV DESCHASHPLUGIN_ARCH=linux-arm64
-ENV DESCHASHPLUGIN_HASH=d48c3e5aede77bd9cb72d78689ce12c0327f624435cb0496b3eacb92df416363
-
-FROM base-downloader AS base-downloader-linux-arm
-ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf
-ENV DESCHASHPLUGIN_ARCH=linux-arm
-ENV DESCHASHPLUGIN_HASH=f7df336c72dd1674bd18ff23862a410b6a9691a3e13752264dcffa0950e21c74
-
-FROM base-downloader-${TARGETOS}-${TARGETARCH} AS downloader
+# This dockerfile is meant to compile a c-lightning x64 image
+# It is using multi stage build:
+# * downloader: Download litecoin/bitcoin and qemu binaries needed for c-lightning
+# * builder: Compile c-lightning dependencies, then c-lightning itself with static linking
+# * final: Copy the binaries required at runtime
+# The resulting image uploaded to dockerhub will only contain what is needed for runtime.
+# From the root of the repository, run "docker build -t yourimage:yourtag ."
+FROM debian:buster-slim as downloader
 
 RUN set -ex \
 	&& apt-get update \
@@ -37,9 +13,12 @@ RUN set -ex \
 
 WORKDIR /opt
 
+RUN wget -qO /opt/tini "https://github.com/krallin/tini/releases/download/v0.18.0/tini" \
+    && echo "12d20136605531b09a2c2dac02ccee85e1b874eb322ef6baf7561cd93f93c855 /opt/tini" | sha256sum -c - \
+    && chmod +x /opt/tini
 
-ENV BITCOIN_VERSION=27.1
-ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
+ARG BITCOIN_VERSION=22.0
+ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
 ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/$BITCOIN_TARBALL
 ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS
 
@@ -49,296 +28,77 @@ RUN mkdir /opt/bitcoin && cd /opt/bitcoin \
     && grep $BITCOIN_TARBALL bitcoin | tee SHA256SUMS \
     && sha256sum -c SHA256SUMS \
     && BD=bitcoin-$BITCOIN_VERSION/bin \
-    && tar -xzvf $BITCOIN_TARBALL $BD/ --strip-components=1 \
+    && tar -xzvf $BITCOIN_TARBALL $BD/bitcoin-cli --strip-components=1 \
     && rm $BITCOIN_TARBALL
 
 ENV LITECOIN_VERSION 0.16.3
-ENV LITECOIN_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
+ENV LITECOIN_PGP_KEY FE3348877809386C
+ENV LITECOIN_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-x86_64-linux-gnu.tar.gz
+ENV LITECOIN_ASC_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-linux-signatures.asc
+ENV LITECOIN_SHA256 686d99d1746528648c2c54a1363d046436fd172beadaceea80bdc93043805994
 
 # install litecoin binaries
 RUN mkdir /opt/litecoin && cd /opt/litecoin \
     && wget -qO litecoin.tar.gz "$LITECOIN_URL" \
-    && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \
+    && echo "$LITECOIN_SHA256  litecoin.tar.gz" | sha256sum -c - \
+    && BD=litecoin-$LITECOIN_VERSION/bin \
+    && tar -xzvf litecoin.tar.gz $BD/litecoin-cli --strip-components=1 --exclude=*-qt \
     && rm litecoin.tar.gz
 
-ENV DESCHASHPLUGIN_URL https://github.com/nbd-wtf/invoicewithdescriptionhash/releases/download/v1.4/invoicewithdescriptionhash-v1.4-${DESCHASHPLUGIN_ARCH}.tar.gz
-ENV DESCHASHPLUGIN_SHA256 ${DESCHASHPLUGIN_HASH}
-RUN mkdir /opt/deschashplugin && cd /opt/deschashplugin \
-    && wget -qO invoicewithdescriptionhash.tar.gz "$DESCHASHPLUGIN_URL" \
-    && echo "$DESCHASHPLUGIN_SHA256  invoicewithdescriptionhash.tar.gz" | sha256sum -c - \
-    && tar -xzvf invoicewithdescriptionhash.tar.gz && rm invoicewithdescriptionhash.tar.gz \
-    && chmod a+x invoicewithdescriptionhash
+FROM debian:buster-slim as builder
 
-FROM --platform=${DEFAULT_TARGETPLATFORM} ${BASE_DISTRO} AS base-builder
-RUN apt-get update -qq && \
-    apt-get install -qq -y --no-install-recommends \
-        autoconf \
-        automake \
-        bison \
-        build-essential \
-        ca-certificates \
-        curl \
-        dirmngr \
-        flex \
-        gettext \
-        git \
-        gnupg \
-        jq \
-        libicu-dev \
-        libtool \
-        libffi-dev \
-        pkg-config \
-        libssl-dev \
-        protobuf-compiler \
-        python3 \
-        python3-dev \
-        python3-mako \
-        python3-pip \
-        python3-venv \
-        python3-setuptools \
-        libev-dev \
-        libevent-dev \
-        qemu-user-static \
-        wget \
-        unzip \
-        tclsh
+ENV LIGHTNINGD_VERSION=master
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates autoconf automake build-essential git libtool python3 python3-pip python3-setuptools python3-mako wget gnupg dirmngr git gettext
 
-ENV PATH="/root/.local/bin:$PATH" \
-    PYTHON_VERSION=3 \
-    POETRY_VERSION=2.0.1
-RUN curl -sSL https://install.python-poetry.org | python3 - && \
-    poetry self add poetry-plugin-export
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install --upgrade pip setuptools wheel
+RUN wget -q https://zlib.net/zlib-1.2.11.tar.gz \
+&& tar xvf zlib-1.2.11.tar.gz \
+&& cd zlib-1.2.11 \
+&& ./configure \
+&& make \
+&& make install && cd .. && rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11
 
-RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz -O zlib.tar.gz && \
-    wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip -O sqlite.zip && \
-    wget -q https://ftp.postgresql.org/pub/source/v17.1/postgresql-17.1.tar.gz -O postgres.tar.gz
+RUN apt-get install -y --no-install-recommends unzip tclsh \
+&& wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip \
+&& unzip sqlite-src-3290000.zip \
+&& cd sqlite-src-3290000 \
+&& ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension \
+&& make \
+&& make install && cd .. && rm sqlite-src-3290000.zip && rm -rf sqlite-src-3290000
+
+RUN wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz \
+&& tar xvf gmp-6.1.2.tar.xz \
+&& cd gmp-6.1.2 \
+&& ./configure --disable-assembly \
+&& make \
+&& make install && cd .. && rm gmp-6.1.2.tar.xz && rm -rf gmp-6.1.2
 
 WORKDIR /opt/lightningd
 COPY . /tmp/lightning
 RUN git clone --recursive /tmp/lightning . && \
     git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD)
 
-# Do not build python plugins (wss-proxy) here, python doesn't support cross compilation.
-RUN sed -i '/^wss-proxy/d' pyproject.toml && \
-    poetry lock && \
-    poetry export -o requirements.txt --without-hashes
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install -r requirements.txt && \
-    pip3 cache purge
-WORKDIR /
-
-FROM base-builder AS base-builder-linux-amd64
-
-ENV POSTGRES_CONFIG="--without-readline" \
-    PG_CONFIG=/usr/local/pgsql/bin/pg_config
-
-FROM base-builder AS base-builder-linux-arm64
-ENV target_host=aarch64-linux-gnu \
-    target_host_rust=aarch64-unknown-linux-gnu \
-    target_host_qemu=qemu-aarch64-static
-
-RUN apt-get install -qq -y --no-install-recommends \
-        libc6-arm64-cross \
-        gcc-${target_host} \
-        g++-${target_host}
-
-ENV AR=${target_host}-ar \
-    AS=${target_host}-as \
-    CC=${target_host}-gcc \
-    CXX=${target_host}-g++ \
-    LD=${target_host}-ld \
-    STRIP=${target_host}-strip \
-    QEMU_LD_PREFIX=/usr/${target_host} \
-    HOST=${target_host} \
-    TARGET=${target_host_rust} \
-    RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
-    PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
-
-ENV ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
-    SQLITE_CONFIG="--host=${target_host} --prefix=${QEMU_LD_PREFIX}" \
-    POSTGRES_CONFIG="--without-readline --prefix=${QEMU_LD_PREFIX}" \
-    PG_CONFIG="${QEMU_LD_PREFIX}/bin/pg_config"
-
-FROM base-builder AS base-builder-linux-arm
-
-ENV target_host=arm-linux-gnueabihf \
-    target_host_rust=armv7-unknown-linux-gnueabihf \
-    target_host_qemu=qemu-arm-static
-
-RUN apt-get install -qq -y --no-install-recommends \
-        libc6-armhf-cross \
-        gcc-${target_host} \
-        g++-${target_host}
-
-ENV AR=${target_host}-ar \
-    AS=${target_host}-as \
-    CC=${target_host}-gcc \
-    CXX=${target_host}-g++ \
-    LD=${target_host}-ld \
-    STRIP=${target_host}-strip \
-    QEMU_LD_PREFIX=/usr/${target_host} \
-    HOST=${target_host} \
-    TARGET=${target_host_rust} \
-    RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
-    PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
-
-ENV ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
-    SQLITE_CONFIG="--host=${target_host} --prefix=${QEMU_LD_PREFIX}" \
-    POSTGRES_CONFIG="--without-readline --prefix=${QEMU_LD_PREFIX}" \
-    PG_CONFIG="${QEMU_LD_PREFIX}/bin/pg_config"
-
-FROM base-builder-${TARGETOS}-${TARGETARCH} AS builder
-
-ENV LIGHTNINGD_VERSION=master
-
-RUN mkdir zlib && tar xvf zlib.tar.gz -C zlib --strip-components=1 \
-    && cd zlib \
-    && ./configure ${ZLIB_CONFIG} \
-    && make \
-    && make install && cd .. && \
-    rm zlib.tar.gz && \
-    rm -rf zlib
-
-RUN unzip sqlite.zip \
-    && cd sqlite-* \
-    && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension ${SQLITE_CONFIG} \
-    && make \
-    && make install && cd .. && rm sqlite.zip && rm -rf sqlite-*
-
-RUN mkdir postgres && tar xvf postgres.tar.gz -C postgres --strip-components=1 \
-    && cd postgres \
-    && ./configure ${POSTGRES_CONFIG} \
-    && cd src/include \
-    && make install \
-    && cd ../interfaces/libpq \
-    && make install \
-    && cd ../../bin/pg_config \
-    && make install \
-    && cd ../../../../ && \
-    rm postgres.tar.gz && \
-    rm -rf postgres && \
-    ldconfig "$(${PG_CONFIG} --libdir)"
-
-# Save libpq to a specific location to copy it into the final image.
-RUN mkdir /var/libpq && cp -a "$(${PG_CONFIG} --libdir)"/libpq.* /var/libpq
-
-ENV RUST_PROFILE=release \
-    PATH="/root/.cargo/bin:/root/.local/bin:$PATH"
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
-RUN rustup toolchain install stable --component rustfmt --allow-downgrade
-
-COPY --from=downloader /usr/bin/${target_host_qemu} /usr/bin/${target_host_qemu}
-WORKDIR /opt/lightningd
-
-# If cross-compiling, need to tell it to cargo.
-RUN ( ! [ -n "${target_host}" ] ) || \
-    (mkdir -p .cargo && echo "[target.${target_host_rust}]\nlinker = \"${target_host}-gcc\"" > .cargo/config)
-
-# Weird errors with cargo for cln-grpc on arm7 https://github.com/ElementsProject/lightning/issues/6596
-RUN ( ! [ "${target_host}" = "arm-linux-gnueabihf" ] ) || \
-    (sed -i '/documentation = "https:\/\/docs.rs\/cln-grpc"/a include = ["**\/*.*"]' cln-grpc/Cargo.toml)
-
-# Ensure that the desired grpcio-tools & protobuf versions are installed
-# https://github.com/ElementsProject/lightning/pull/7376#issuecomment-2161102381
-RUN poetry lock && poetry install && \
-    poetry self add poetry-plugin-export
-
-# Ensure that git differences are removed before making bineries, to avoid `-modded` suffix
-# poetry.lock changed due to pyln-client, pyln-proto and pyln-testing version updates
-# pyproject.toml was updated to exclude wss-proxy plugins in base-builder stage
-RUN git reset --hard HEAD
-
-RUN ./configure --prefix=/tmp/lightning_install --enable-static && poetry run make install
-
-# Export the requirements for the plugins so we can install them in builder-python stage
-WORKDIR /opt/lightningd/plugins/wss-proxy
-RUN poetry lock && poetry export -o requirements.txt --without-hashes
-WORKDIR /opt/lightningd
-RUN echo 'RUSTUP_INSTALL_OPTS="${RUSTUP_INSTALL_OPTS}"' > /tmp/rustup_install_opts.txt
-
-# We need to build python plugins on the target's arch because python doesn't support cross build
-FROM ${BASE_DISTRO} AS builder-python
-RUN apt-get update -qq && \
-    apt-get install -qq -y --no-install-recommends \
-        git \
-        curl \
-        libtool \
-        pkg-config \
-        autoconf \
-        automake \
-        build-essential \
-        libffi-dev \
-        libssl-dev \
-        python3 \
-        python3-dev \
-        python3-pip \
-        python3-venv && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
+ARG DEVELOPER=0
 ENV PYTHON_VERSION=3
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install --upgrade pip setuptools wheel
+RUN pip3 install mrkd
+RUN ./configure --prefix=/tmp/lightning_install --enable-static && make -j3 DEVELOPER=${DEVELOPER} && make install
 
-# Copy rustup_install_opts.txt file from builder
-COPY --from=builder /tmp/rustup_install_opts.txt /tmp/rustup_install_opts.txt
-# Setup ENV $RUSTUP_INSTALL_OPTS for this stage
-RUN export $(cat /tmp/rustup_install_opts.txt)
-ENV PATH="/root/.cargo/bin:/root/.venvs/cln/bin:$PATH"
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
+FROM debian:buster-slim as final
 
-WORKDIR /opt/lightningd/plugins/wss-proxy
-COPY --from=builder /opt/lightningd/plugins/wss-proxy/requirements.txt .
-RUN pip3 install -r requirements.txt
-RUN pip3 cache purge
+COPY --from=downloader /opt/tini /usr/bin/tini
+RUN apt-get update && apt-get install -y --no-install-recommends socat inotify-tools python3 python3-pip \
+    && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /opt/lightningd
-
-FROM ${BASE_DISTRO} AS final
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-      tini \
-      socat \
-      inotify-tools \
-      jq \
-      python3 \
-      python3-pip && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-ENV LIGHTNINGD_DATA=/root/.lightning \
-    LIGHTNINGD_RPC_PORT=9835 \
-    LIGHTNINGD_PORT=9735 \
-    LIGHTNINGD_NETWORK=bitcoin
+ENV LIGHTNINGD_DATA=/root/.lightning
+ENV LIGHTNINGD_RPC_PORT=9835
+ENV LIGHTNINGD_PORT=9735
+ENV LIGHTNINGD_NETWORK=bitcoin
 
 RUN mkdir $LIGHTNINGD_DATA && \
-    mkdir /etc/bundledplugins && \
-    mkdir $LIGHTNINGD_DATA/plugins && \
     touch $LIGHTNINGD_DATA/config
 VOLUME [ "/root/.lightning" ]
-
-# Take libpq directly from builder.
-RUN mkdir /var/libpq && mkdir -p /usr/local/pgsql/lib
-RUN --mount=type=bind,from=builder,source=/var/libpq,target=/var/libpq,rw \
-    cp -a /var/libpq/libpq.* /usr/local/pgsql/lib && \
-    echo "/usr/local/pgsql/lib" > /etc/ld.so.conf.d/libpq.conf && \
-    ldconfig
-
 COPY --from=builder /tmp/lightning_install/ /usr/local/
-COPY --from=builder-python /root/.venvs/cln/lib/python3.11/site-packages /usr/local/lib/python3.11/dist-packages/
 COPY --from=downloader /opt/bitcoin/bin /usr/bin
 COPY --from=downloader /opt/litecoin/bin /usr/bin
-COPY --from=downloader /opt/deschashplugin $LIGHTNINGD_DATA/plugins
-COPY --from=downloader /opt/deschashplugin /etc/bundledplugins
 COPY tools/docker-entrypoint.sh entrypoint.sh
 
 EXPOSE 9735 9835

LICENSE

@@ -1,7 +1,7 @@
 Note: the modules in the ccan/ directory have their own licenses, but
 the rest of the code is covered by the following (BSD-MIT) license:
 
-	Copyright Rusty Russell (Blockstream) 2015-2024.
+	Copyright Rusty Russell (Blockstream) 2015.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,10 +1,11 @@
 #! /usr/bin/make
 
 # Extract version from git, or if we're from a zipfile, use dirname
-VERSION=$(shell git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|v\1|gp')
+VERSION=$(shell git describe --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|\1|gp')
 
-# Next release.
-CLN_NEXT_VERSION := v25.05
+ifeq ($(VERSION),)
+$(error "ERROR: git is required for generating version information")
+endif
 
 # --quiet / -s means quiet, dammit!
 ifeq ($(findstring s,$(word 1, $(MAKEFLAGS))),s)
@@ -16,17 +17,14 @@ SUPPRESS_OUTPUT :=
 endif
 
 DISTRO=$(shell lsb_release -is 2>/dev/null || echo unknown)-$(shell lsb_release -rs 2>/dev/null || echo unknown)
-OS=$(shell uname -s)
-ARCH=$(shell uname -m)
-# Changing this could break installs!
 PKGNAME = c-lightning
 
 # We use our own internal ccan copy.
 CCANDIR := ccan
 
 # Where we keep the BOLT RFCs
-BOLTDIR := ../bolts/
-DEFAULT_BOLTVERSION := ccfa38ed4f592c3711156bb4ded77f44ec01101d
+BOLTDIR := ../lightning-rfc/
+DEFAULT_BOLTVERSION := 498f104fd399488c77f449d05cb21c0b604636a2
 # Can be overridden on cmdline.
 BOLTVERSION := $(DEFAULT_BOLTVERSION)
 
@@ -36,7 +34,7 @@ SORT=LC_ALL=C sort
 
 
 ifeq ($V,1)
-VERBOSE = $(ECHO) '$(subst ','\'',$(2))'; $(2)
+VERBOSE = $(ECHO) '$(2)'; $(2)
 else
 VERBOSE = $(ECHO) $(1); $(2)
 endif
@@ -46,7 +44,21 @@ VG=VALGRIND=1 valgrind -q --error-exitcode=7
 VG_TEST_ARGS = --track-origins=yes --leak-check=full --show-reachable=yes --errors-for-leak-kinds=all
 endif
 
-ifeq ($(DEBUGBUILD),1)
+SANITIZER_FLAGS :=
+
+ifneq ($(ASAN),0)
+SANITIZER_FLAGS += -fsanitize=address
+endif
+
+ifneq ($(UBSAN),0)
+SANITIZER_FLAGS += -fsanitize=undefined
+endif
+
+ifneq ($(FUZZING), 0)
+SANITIZER_FLAGS += -fsanitize=fuzzer-no-link
+endif
+
+ifeq ($(DEVELOPER),1)
 DEV_CFLAGS=-DCCAN_TAKE_DEBUG=1 -DCCAN_TAL_DEBUG=1 -DCCAN_JSON_OUT_DEBUG=1
 else
 DEV_CFLAGS=
@@ -56,10 +68,6 @@ ifeq ($(COVERAGE),1)
 COVFLAGS = --coverage
 endif
 
-ifeq ($(CLANG_COVERAGE),1)
-COVFLAGS+=-fprofile-instr-generate -fcoverage-mapping
-endif
-
 ifeq ($(PIE),1)
 PIE_CFLAGS=-fPIE -fPIC
 PIE_LDFLAGS=-pie
@@ -67,25 +75,24 @@ endif
 
 ifeq ($(COMPAT),1)
 # We support compatibility with pre-0.6.
-COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1 -DCOMPAT_V061=1 -DCOMPAT_V062=1 -DCOMPAT_V070=1 -DCOMPAT_V072=1 -DCOMPAT_V073=1 -DCOMPAT_V080=1 -DCOMPAT_V081=1 -DCOMPAT_V082=1 -DCOMPAT_V090=1 -DCOMPAT_V0100=1 -DCOMPAT_V0121=1
+COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1 -DCOMPAT_V061=1 -DCOMPAT_V062=1 -DCOMPAT_V070=1 -DCOMPAT_V072=1 -DCOMPAT_V073=1 -DCOMPAT_V080=1 -DCOMPAT_V081=1 -DCOMPAT_V082=1 -DCOMPAT_V090=1 -DCOMPAT_V0100=1
 endif
 
 # (method=thread to support xdist)
 PYTEST_OPTS := -v -p no:logging $(PYTEST_OPTS)
-MY_CHECK_PYTHONPATH=$${PYTHONPATH}$${PYTHONPATH:+:}$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7:$(shell pwd)/contrib/pyln-grpc-proto
+MY_CHECK_PYTHONPATH=$${PYTHONPATH}$${PYTHONPATH:+:}$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/external/lnprototest:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7
 # Collect generated python files to be excluded from lint checks
-PYTHON_GENERATED= \
-	contrib/pyln-grpc-proto/pyln/grpc/primitives_pb2.py \
-	contrib/pyln-grpc-proto/pyln/grpc/node_pb2_grpc.py \
-	contrib/pyln-grpc-proto/pyln/grpc/node_pb2.py \
-	contrib/pyln-testing/pyln/testing/grpc2py.py
+PYTHON_GENERATED=
+
+# Options to pass to cppcheck. Mostly used to exclude files that are
+# generated with external tools that we don't have control over
+CPPCHECK_OPTS=-q --language=c --std=c11 --error-exitcode=1 --suppressions-list=.cppcheck-suppress --inline-suppr
 
 # This is where we add new features as bitcoin adds them.
 FEATURES :=
 
 CCAN_OBJS :=					\
 	ccan-asort.o				\
-	ccan-base64.o				\
 	ccan-bitmap.o				\
 	ccan-bitops.o				\
 	ccan-breakpoint.o			\
@@ -121,8 +128,6 @@ CCAN_OBJS :=					\
 	ccan-ptr_valid.o			\
 	ccan-rbuf.o				\
 	ccan-read_write_all.o			\
-	ccan-rune-coding.o			\
-	ccan-rune-rune.o			\
 	ccan-str-base32.o			\
 	ccan-str-hex.o				\
 	ccan-str.o				\
@@ -165,7 +170,6 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/endian/endian.h			\
 	$(CCANDIR)/ccan/err/err.h			\
 	$(CCANDIR)/ccan/fdpass/fdpass.h			\
-	$(CCANDIR)/ccan/graphql/graphql.h		\
 	$(CCANDIR)/ccan/htable/htable.h			\
 	$(CCANDIR)/ccan/htable/htable_type.h		\
 	$(CCANDIR)/ccan/ilog/ilog.h			\
@@ -180,7 +184,6 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/json_out/json_out.h		\
 	$(CCANDIR)/ccan/likely/likely.h			\
 	$(CCANDIR)/ccan/list/list.h			\
-	$(CCANDIR)/ccan/lqueue/lqueue.h			\
 	$(CCANDIR)/ccan/mem/mem.h			\
 	$(CCANDIR)/ccan/membuf/membuf.h			\
 	$(CCANDIR)/ccan/noerr/noerr.h			\
@@ -192,8 +195,6 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/ptrint/ptrint.h			\
 	$(CCANDIR)/ccan/rbuf/rbuf.h			\
 	$(CCANDIR)/ccan/read_write_all/read_write_all.h	\
-	$(CCANDIR)/ccan/rune/internal.h			\
-	$(CCANDIR)/ccan/rune/rune.h			\
 	$(CCANDIR)/ccan/short_types/short_types.h	\
 	$(CCANDIR)/ccan/str/base32/base32.h		\
 	$(CCANDIR)/ccan/str/hex/hex.h			\
@@ -225,46 +226,12 @@ WIRE_GEN_DEPS := $(WIRE_GEN) $(wildcard tools/gen/*_template)
 # These are filled by individual Makefiles
 ALL_PROGRAMS :=
 ALL_TEST_PROGRAMS :=
-ALL_TEST_GEN :=
 ALL_FUZZ_TARGETS :=
 ALL_C_SOURCES :=
-ALL_C_HEADERS :=
-# Extra (non C) targets that should be built by default.
-DEFAULT_TARGETS :=
-
-# Installation directories
-exec_prefix = $(PREFIX)
-bindir = $(exec_prefix)/bin
-libexecdir = $(exec_prefix)/libexec
-pkglibexecdir = $(libexecdir)/$(PKGNAME)
-plugindir = $(pkglibexecdir)/plugins
-datadir = $(PREFIX)/share
-docdir = $(datadir)/doc/$(PKGNAME)
-mandir = $(datadir)/man
-man1dir = $(mandir)/man1
-man5dir = $(mandir)/man5
-man7dir = $(mandir)/man7
-man8dir = $(mandir)/man8
-
-# M1 macos machines with homebrew will install the native libraries in
-# /opt/homebrew instead of /usr/local, most likely because they
-# emulate x86_64 compatibility via Rosetta, and wanting to keep the
-# libraries separate. This however means we also need to switch out
-# the paths accordingly when we detect we're on an M1 macos machine.
-ifeq ("$(OS)-$(ARCH)", "Darwin-arm64")
-CPATH := /opt/homebrew/include
-LIBRARY_PATH := /opt/homebrew/lib
-LDFLAGS := -L/opt/homebrew/opt/sqlite/lib
-CPPFLAGS := -I/opt/homebrew/opt/sqlite/include
-PKG_CONFIG_PATH=/opt/homebrew/opt/sqlite/lib/pkgconfig
-else
-CPATH := /usr/local/include
-LIBRARY_PATH := /usr/local/lib
-endif
-
-CPPFLAGS += -DCLN_NEXT_VERSION="\"$(CLN_NEXT_VERSION)\"" -DPKGLIBEXECDIR="\"$(pkglibexecdir)\"" -DBINDIR="\"$(bindir)\"" -DPLUGINDIR="\"$(plugindir)\"" -DCCAN_TAL_NEVER_RETURN_NULL=1
-CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I$(CPATH) $(SQLITE3_CFLAGS) $(SODIUM_CFLAGS) $(POSTGRES_INCLUDE) $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) $(CSANFLAGS)
+ALL_C_HEADERS := header_versions_gen.h version_gen.h
 
+CPPFLAGS += -DBINTOPKGLIBEXECDIR="\"$(shell sh tools/rel.sh $(bindir) $(pkglibexecdir))\""
+CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I/usr/local/include $(SQLITE3_CFLAGS) $(POSTGRES_INCLUDE) $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) -DBUILD_ELEMENTS=1
 # If CFLAGS is already set in the environment of make (to whatever value, it
 # does not matter) then it would export it to subprocesses with the above value
 # we set, including CWARNFLAGS which by default contains -Wall -Werror. This
@@ -275,15 +242,16 @@ unexport CFLAGS
 # We can get configurator to run a different compile cmd to cross-configure.
 CONFIGURATOR_CC := $(CC)
 
-LDFLAGS += $(PIE_LDFLAGS) $(CSANFLAGS) $(COPTFLAGS)
+LDFLAGS += $(PIE_LDFLAGS) $(SANITIZER_FLAGS) $(COPTFLAGS)
+CFLAGS += $(SANITIZER_FLAGS)
 
 ifeq ($(STATIC),1)
 # For MacOS, Jacob Rapoport <jacob@rumblemonkey.com> changed this to:
-#  -L/usr/local/lib -lsqlite3 -lz -Wl,-lm -lpthread -ldl $(COVFLAGS)
+#  -L/usr/local/lib -Wl,-lgmp -lsqlite3 -lz -Wl,-lm -lpthread -ldl $(COVFLAGS)
 # But that doesn't static link.
-LDLIBS = -L$(CPATH) -Wl,-dn $(SQLITE3_LDLIBS) -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
+LDLIBS = -L/usr/local/lib -Wl,-dn -lgmp $(SQLITE3_LDLIBS) -lz -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
 else
-LDLIBS = -L$(CPATH) -lm $(SQLITE3_LDLIBS) $(COVFLAGS)
+LDLIBS = -L/usr/local/lib -lm -lgmp $(SQLITE3_LDLIBS) -lz $(COVFLAGS)
 endif
 
 # If we have the postgres client library we need to link against it as well
@@ -291,25 +259,35 @@ ifeq ($(HAVE_POSTGRES),1)
 LDLIBS += $(POSTGRES_LDLIBS)
 endif
 
-default: show-flags gen all-programs all-test-programs doc-all default-targets $(PYTHON_GENERATED)
+default: show-flags all-programs all-test-programs doc-all
 
 ifneq ($(SUPPRESS_GENERATION),1)
 FORCE = FORCE
-FORCE:
+FORCE::
 endif
 
 show-flags: config.vars
 	@$(ECHO) "CC: $(CC) $(CFLAGS) -c -o"
 	@$(ECHO) "LD: $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) -o"
 
-# We will re-generate, but we won't generate for the first time!
-ccan/config.h config.vars &: configure ccan/tools/configurator/configurator.c
-	@if [ ! -f config.vars ]; then echo 'File config.vars not found: you must run ./configure before running make.' >&2; exit 1; fi
+ccan/config.h: config.vars configure ccan/tools/configurator/configurator.c
 	./configure --reconfigure
 
+config.vars:
+	@echo 'File config.vars not found: you must run ./configure before running make.' >&2
+	@exit 1
+
 %.o: %.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 
+# '_exp' inserted before _wiregen.[ch] to demark experimental
+# spec-derived headers, which are *not* committed into git.
+ifeq ($(EXPERIMENTAL_FEATURES),1)
+EXP := _exp
+else
+EXP :=
+endif
+
 # tools/update-mocks.sh does nasty recursive make, must not do this!
 ifeq ($(SUPPRESS_GENERATION),1)
 SHA256STAMP_CHANGED = false
@@ -344,18 +322,10 @@ endif
 		$(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \
 	fi
 
-RUST_PROFILE ?= debug
-ifneq ($(RUST_PROFILE),debug)
-CARGO_OPTS := --profile=$(RUST_PROFILE) --quiet
-else
-CARGO_OPTS := --quiet
-endif
-
 include external/Makefile
 include bitcoin/Makefile
 include common/Makefile
 include wire/Makefile
-include db/Makefile
 include hsmd/Makefile
 include gossipd/Makefile
 include openingd/Makefile
@@ -366,46 +336,15 @@ include connectd/Makefile
 include lightningd/Makefile
 include cli/Makefile
 include doc/Makefile
-include contrib/msggen/Makefile
 include devtools/Makefile
 include tools/Makefile
-ifneq ($(RUST),0)
-include cln-rpc/Makefile
-include cln-grpc/Makefile
-endif
 include plugins/Makefile
 include tests/plugins/Makefile
-
+include contrib/libhsmd_python/Makefile
 ifneq ($(FUZZING),0)
 	include tests/fuzz/Makefile
 endif
 
-ifneq ($V,1)
-MSGGEN_ARGS := -s
-endif
-
-$(MSGGEN_GENALL)&: contrib/msggen/msggen/schema.json
-	@$(call VERBOSE, "msggen $@", PYTHONPATH=contrib/msggen $(PYTHON) contrib/msggen/msggen/__main__.py $(MSGGEN_ARGS) generate)
-
-# The compiler assumes that the proto files are in the same
-# directory structure as the generated files will be. Since we
-# don't do that we need to path the files up.
-GRPC_DIR = contrib/pyln-grpc-proto/pyln
-GRPC_PATH = $(GRPC_DIR)/grpc
-
-GRPC_GEN = \
-	$(GRPC_PATH)/node_pb2.py \
-	$(GRPC_PATH)/node_pb2_grpc.py \
-	$(GRPC_PATH)/primitives_pb2.py
-
-ALL_TEST_GEN += $(GRPC_GEN)
-
-$(GRPC_GEN) &: cln-grpc/proto/node.proto cln-grpc/proto/primitives.proto
-	$(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/node.proto --python_out=$(GRPC_PATH)/ --grpc_python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
-	$(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/primitives.proto --python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
-	find $(GRPC_DIR)/ -type f -name "*.py" -print0 | xargs -0 sed -i'.bak' -e 's/^import \(.*\)_pb2 as .*__pb2/from pyln.grpc import \1_pb2 as \1__pb2/g'
-	find $(GRPC_DIR)/ -type f -name "*.py.bak" -print0 | xargs -0 rm -f
-
 # We make pretty much everything depend on these.
 ALL_GEN_HEADERS := $(filter %gen.h,$(ALL_C_HEADERS))
 ALL_GEN_SOURCES := $(filter %gen.c,$(ALL_C_SOURCES))
@@ -419,8 +358,7 @@ ALL_NONGEN_SRCFILES := $(ALL_NONGEN_HEADERS) $(ALL_NONGEN_SOURCES)
 BIN_PROGRAMS = \
 	       cli/lightning-cli \
 	       lightningd/lightningd \
-	       tools/lightning-hsmtool\
-	       tools/reckless
+	       tools/lightning-hsmtool
 PKGLIBEXEC_PROGRAMS = \
 	       lightningd/lightning_channeld \
 	       lightningd/lightning_closingd \
@@ -432,18 +370,8 @@ PKGLIBEXEC_PROGRAMS = \
 	       lightningd/lightning_openingd \
 	       lightningd/lightning_websocketd
 
-mkdocs.yml: $(MANPAGES:=.md)
-	@$(call VERBOSE, "genidx $@", \
-	  find doc -maxdepth 1 -name '*\.[0-9]\.md' | \
-	  cut -b 5- | LC_ALL=C sort | \
-	  sed 's/\(.*\)\.\(.*\).*\.md/- "\1": "\1.\2.md"/' | \
-	  $(PYTHON) devtools/blockreplace.py mkdocs.yml manpages --language=yml --indent "          " \
-	)
-
-
-
 # Don't delete these intermediaries.
-.PRECIOUS: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) $(PYTHON_GENERATED)
+.PRECIOUS: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES)
 
 # Every single object file.
 ALL_OBJS := $(ALL_C_SOURCES:.c=.o)
@@ -466,29 +394,28 @@ else
 PYTEST_OPTS += -x
 endif
 
-# Allow for targeting specific tests by setting the PYTEST_TESTS environment variable.
-ifeq ($(PYTEST_TESTS),)
-PYTEST_TESTS = "tests/"
-endif
-
 check-units:
 
-check: check-units installcheck pytest
+check: check-units installcheck check-protos pytest
 
-pytest: $(ALL_PROGRAMS) $(DEFAULT_TARGETS) $(ALL_TEST_PROGRAMS) $(ALL_TEST_GEN)
+check-protos: $(ALL_PROGRAMS)
+ifeq ($(PYTEST),)
+	@echo "py.test is required to run the protocol tests, please install using 'pip3 install -r requirements.txt', and rerun 'configure'."; false
+else
+ifeq ($(DEVELOPER),1)
+	@(cd external/lnprototest && PYTHONPATH=$(MY_CHECK_PYTHONPATH) LIGHTNING_SRC=../.. $(PYTEST) --runner lnprototest.clightning.Runner $(PYTEST_OPTS))
+else
+	@echo "lnprototest target requires DEVELOPER=1, skipping"
+endif
+endif
+
+pytest: $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS)
 ifeq ($(PYTEST),)
 	@echo "py.test is required to run the integration tests, please install using 'pip3 install -r requirements.txt', and rerun 'configure'."
 	exit 1
 else
-# Explicitly hand VALGRIND so you can override on make cmd line.
-	PYTHONPATH=$(MY_CHECK_PYTHONPATH) TEST_DEBUG=1 VALGRIND=$(VALGRIND) $(PYTEST) $(PYTEST_TESTS) $(PYTEST_OPTS)
-endif
-
-check-fuzz: $(ALL_FUZZ_TARGETS)
-ifneq ($(FUZZING),0)
-	@tests/fuzz/check-fuzz.sh
-else
-	@echo "fuzzing is not enabled: first run './configure --enable-fuzzing'"
+# Explicitly hand DEVELOPER and VALGRIND so you can override on make cmd line.
+	PYTHONPATH=$(MY_CHECK_PYTHONPATH) TEST_DEBUG=1 DEVELOPER=$(DEVELOPER) VALGRIND=$(VALGRIND) $(PYTEST) tests/ $(PYTEST_OPTS)
 endif
 
 # Keep includes in alpha order.
@@ -533,10 +460,13 @@ check-whitespace/%: %
 
 check-whitespace: check-whitespace/Makefile check-whitespace/tools/check-bolt.c $(ALL_NONGEN_SRCFILES:%=check-whitespace/%)
 
+check-markdown:
+	@tools/check-markdown.sh
+
 check-spelling:
 	@tools/check-spelling.sh
 
-PYSRC=$(shell git ls-files "*.py" | grep -v /text.py)
+PYSRC=$(shell git ls-files "*.py" | grep -v /text.py) contrib/pylightning/lightning-pay
 
 # Some tests in pyln will need to find lightningd to run, so have a PATH that
 # allows it to find that
@@ -551,7 +481,7 @@ check-python-flake8:
 	@# E731 do not assign a lambda expression, use a def
 	@# W503: line break before binary operator
 	@# E741: ambiguous variable name
-	@flake8 --ignore=E501,E731,E741,W503,F541,E275 --exclude $(shell echo ${PYTHON_GENERATED} | sed 's/ \+/,/g') ${PYSRC}
+	@flake8 --ignore=E501,E731,E741,W503 --exclude $(shell echo ${PYTHON_GENERATED} | sed 's/ \+/,/g') ${PYSRC}
 
 check-pytest-pyln-proto:
 	PATH=$(PYLN_PATH) PYTHONPATH=$(MY_CHECK_PYTHONPATH) $(PYTEST) contrib/pyln-proto/tests/
@@ -559,8 +489,15 @@ check-pytest-pyln-proto:
 check-includes: check-src-includes check-hdr-includes
 	@tools/check-includes.sh
 
+# cppcheck gets confused by list_for_each(head, i, list): thinks i is uninit.
+.cppcheck-suppress:
+	@git ls-files -- "*.c" "*.h" | grep -vE '^(ccan|contrib)/' | xargs grep -n '_for_each' | sed 's/\([^:]*:.*\):.*/uninitvar:\1/' > $@
+
+check-cppcheck: .cppcheck-suppress
+	@trap 'rm -f .cppcheck-suppress' 0; git ls-files -- "*.c" "*.h" | grep -vE '^ccan/' | xargs cppcheck  ${CPPCHECK_OPTS}
+
 check-shellcheck:
-	@git ls-files -z -- "*.sh" | xargs -0 shellcheck -f gcc
+	@git ls-files -- "*.sh" | xargs shellcheck
 
 check-setup_locale:
 	@tools/check-setup_locale.sh
@@ -575,29 +512,9 @@ check-discouraged-functions:
 # since it risks overflow.
 check-amount-access:
 	@! (git grep -nE "(->|\.)(milli)?satoshis" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" | grep -v '/* Raw:')
-	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" | grep -vE "sizeof.struct amount_(m)?sat."
+	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*"
 
-repeat-doc-examples:
-	@for i in $$(seq 1 $(n)); do \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-		echo "Iteration $$i" >> tests/autogenerate-examples-repeat.log; \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-		VALGRIND=0 TIMEOUT=40 TEST_DEBUG=1 GENERATE_EXAMPLES=1 pytest -vvv tests/autogenerate-rpc-examples.py; \
-		git diff >> tests/autogenerate-examples-repeat.log; \
-		git reset --hard; \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-	done
-
-update-doc-examples:
-	TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 $(PYTEST) $(PYTEST_OPTS) --timeout=1200 tests/autogenerate-rpc-examples.py && $(MAKE) $(MSGGEN_GEN_ALL)
-
-check-doc-examples: update-doc-examples
-	git diff --exit-code HEAD
-
-# For those without working cppcheck
-check-source-no-cppcheck: check-makefile check-source-bolt check-whitespace check-spelling check-python check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access
-
-check-source: check-source-no-cppcheck
+check-source: check-makefile check-source-bolt check-whitespace check-markdown check-spelling check-python check-includes check-cppcheck check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access
 
 full-check: check check-source
 
@@ -609,23 +526,9 @@ full-check: check check-source
 #
 # Do not run on your development tree since it will complain if you
 # have a dirty tree.
-CHECK_GEN_ALL = \
-	$(CLN_GRPC_GENALL) \
-	$(CLN_RPC_GENALL) \
-	$(MANPAGES) \
-	$(WALLET_DB_QUERIES) \
-	$(PYTHON_GENERATED) \
-	$(ALL_GEN_HEADERS) \
-	$(ALL_GEN_SOURCES) \
-	$(MSGGEN_GEN_ALL) \
-	wallet/statements_gettextgen.po \
-	doc/index.rst
-
-gen:  $(CHECK_GEN_ALL)
-
-check-gen-updated:  $(CHECK_GEN_ALL)
+check-gen-updated: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) wallet/statements_gettextgen.po $(MANPAGES)
 	@echo "Checking for generated files being changed by make"
-	git diff --exit-code HEAD
+	git diff --exit-code HEAD $?
 
 coverage/coverage.info: check pytest
 	mkdir coverage || true
@@ -641,10 +544,7 @@ ncc: ${TARGET_DIR}/libwally-core-build/src/libwallycore.la
 
 # Ignore test/ directories.
 TAGS:
-	$(RM) TAGS; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 etags --append
-
-tags:
-	$(RM) tags; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 ctags --append
+	$(RM) TAGS; find * -name test -type d -prune -o -name '*.[ch]' -print -o -name '*.py' -print | xargs etags --append
 
 ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS)
 
@@ -652,28 +552,16 @@ ALL_PROGRAMS += ccan/ccan/cdump/tools/cdump-enumstr
 # Can't add to ALL_OBJS, as that makes a circular dep.
 ccan/ccan/cdump/tools/cdump-enumstr.o: $(CCAN_HEADERS) Makefile
 
-# Without a working git, you can't generate this file, so assume if it exists
-# it is ok (fixes "sudo make install").
-ifeq ($(VERSION),)
-version_gen.h:
-	echo "ERROR: git is required for generating version information" >&2
-	exit 1
-else
 version_gen.h: $(FORCE)
 	@(echo "#define VERSION \"$(VERSION)\"" && echo "#define BUILD_FEATURES \"$(FEATURES)\"") > $@.new
 	@if cmp $@.new $@ >/dev/null 2>&1; then rm -f $@.new; else mv $@.new $@; $(ECHO) Version updated; fi
-endif
 
 # That forces this rule to be run every time, too.
-header_versions_gen.h: tools/headerversions $(FORCE)
+header_versions_gen.h: tools/headerversions
 	@tools/headerversions $@
 
-# We make a static library, this way linker can discard unused parts.
-libccan.a: $(CCAN_OBJS)
-	@$(call VERBOSE, "ar $@", $(AR) r $@ $(CCAN_OBJS))
-
 # All binaries require the external libs, ccan and system library versions.
-$(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): $(EXTERNAL_LIBS) libccan.a
+$(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): $(EXTERNAL_LIBS) $(CCAN_OBJS)
 
 # Each test program depends on its own object.
 $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): %: %.o
@@ -683,17 +571,17 @@ $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): %: %.o
 # uses some ccan modules internally).  We want to rely on -lwallycore etc.
 # (as per EXTERNAL_LDLIBS) so we filter them out here.
 $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS):
-	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a $($(@)_LDLIBS) -o $@)
+	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) -o $@)
 
 # We special case the fuzzing target binaries, as they need to link against libfuzzer,
 # which brings its own main().
 FUZZ_LDFLAGS = -fsanitize=fuzzer
 $(ALL_FUZZ_TARGETS):
-	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a $(FUZZ_LDFLAGS) -o $@)
+	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) $(FUZZ_LDFLAGS) -o $@)
 
 
 # Everything depends on the CCAN headers, and Makefile
-$(CCAN_OBJS) $(CDUMP_OBJS): $(CCAN_HEADERS) Makefile ccan_compat.h
+$(CCAN_OBJS) $(CDUMP_OBJS): $(CCAN_HEADERS) Makefile
 
 # Except for CCAN, we treat everything else as dependent on external/ bitcoin/ common/ wire/ and all generated headers, and Makefile
 $(ALL_OBJS): $(BITCOIN_HEADERS) $(COMMON_HEADERS) $(CCAN_HEADERS) $(WIRE_HEADERS) $(ALL_GEN_HEADERS) $(EXTERNAL_HEADERS) Makefile
@@ -714,9 +602,7 @@ update-ccan:
 
 # Now ALL_PROGRAMS is fully populated, we can expand it.
 all-programs: $(ALL_PROGRAMS)
-all-fuzz-programs: $(ALL_FUZZ_TARGETS)
 all-test-programs: $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS)
-default-targets: $(DEFAULT_TARGETS)
 
 distclean: clean
 	$(RM) ccan/config.h config.vars
@@ -724,68 +610,52 @@ distclean: clean
 maintainer-clean: distclean
 	@echo 'This command is intended for maintainers to use; it'
 	@echo 'deletes files that may need special tools to rebuild.'
-	$(RM) $(PYTHON_GENERATED)
 
 # We used to have gen_ files, now we have _gen files.
-# We used to generate doc/schemas/lightning-sql.json.
 obsclean:
 	$(RM) gen_*.h */gen_*.[ch] */*/gen_*.[ch]
-	$(RM) doc/schemas/lightning-sql.json
 
 clean: obsclean
-	$(RM) libccan.a $(CCAN_OBJS) $(CDUMP_OBJS) $(ALL_OBJS)
+	$(RM) $(CCAN_OBJS) $(CDUMP_OBJS) $(ALL_OBJS)
 	$(RM) $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES)
 	$(RM) $(ALL_PROGRAMS)
 	$(RM) $(ALL_TEST_PROGRAMS)
 	$(RM) $(ALL_FUZZ_TARGETS)
-	$(RM) $(MSGGEN_GEN_ALL)
 	$(RM) ccan/tools/configurator/configurator
 	$(RM) ccan/ccan/cdump/tools/cdump-enumstr.o
 	find . -name '*gcda' -delete
 	find . -name '*gcno' -delete
 	find . -name '*.nccout' -delete
-	if [ "${RUST}" -eq "1" ]; then cargo clean; fi
-
-
-PYLNS=client proto testing
-# See doc/contribute-to-core-lightning/contributor-workflow.md
-update-versions: update-pyln-versions update-wss-proxy-version update-poetry-lock update-dot-version update-doc-examples
-
-update-pyln-versions: $(PYLNS:%=update-pyln-version-%)
-
-update-pyln-version-%:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	cd contrib/pyln-$* && $(MAKE) upgrade-version
-
-pyln-release:  $(PYLNS:%=pyln-release-%)
-
-pyln-release-%:
-	cd contrib/pyln-$* && $(MAKE) prod-release
-
-update-wss-proxy-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	cd plugins/wss-proxy && $(MAKE) upgrade-version
-
-update-poetry-lock:
-	poetry update wss-proxy pyln-client pyln-proto pyln-testing update-reckless-version
-
-update-reckless-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	@sed -i "s/__VERSION__ = '\([.-z]*\)'/__VERSION__ = '$(NEW_VERSION)'/" tools/reckless
-
-update-dot-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	echo $(NEW_VERSION) > .version
 
+# These must both be enabled for update-mocks
+ifeq ($(DEVELOPER)$(EXPERIMENTAL_FEATURES),11)
 update-mocks: $(ALL_TEST_PROGRAMS:%=update-mocks/%.c)
+else
+update-mocks:
+	@echo Need DEVELOPER=1 and EXPERIMENTAL_FEATURES=1 to regenerate mocks >&2; exit 1
+endif
 
-$(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) libccan.a ccan/ccan/cdump/tools/cdump-enumstr config.vars
+$(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) $(CCAN_OBJS) ccan/ccan/cdump/tools/cdump-enumstr config.vars
 
-update-mocks/%: % $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES)
+update-mocks/%: %
 	@MAKE=$(MAKE) tools/update-mocks.sh "$*" $(SUPPRESS_OUTPUT)
 
-unittest/%: % bolt-precheck
-	BOLTDIR=$(LOCAL_BOLTDIR) $(VG) $(VG_TEST_ARGS) $* > /dev/null
+unittest/%: %
+	$(VG) $(VG_TEST_ARGS) $* > /dev/null
+
+# Installation directories
+exec_prefix = $(PREFIX)
+bindir = $(exec_prefix)/bin
+libexecdir = $(exec_prefix)/libexec
+pkglibexecdir = $(libexecdir)/$(PKGNAME)
+plugindir = $(pkglibexecdir)/plugins
+datadir = $(PREFIX)/share
+docdir = $(datadir)/doc/$(PKGNAME)
+mandir = $(datadir)/man
+man1dir = $(mandir)/man1
+man5dir = $(mandir)/man5
+man7dir = $(mandir)/man7
+man8dir = $(mandir)/man8
 
 # Commands
 MKDIR_P = mkdir -p
@@ -815,19 +685,17 @@ installdirs:
 
 # $(PLUGINS) is defined in plugins/Makefile.
 
-install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS)
+install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS)
 	@$(NORMAL_INSTALL)
 	$(INSTALL_PROGRAM) $(BIN_PROGRAMS) $(DESTDIR)$(bindir)
 	$(INSTALL_PROGRAM) $(PKGLIBEXEC_PROGRAMS) $(DESTDIR)$(pkglibexecdir)
-	@if [ -d "$(DESTDIR)$(plugindir)/clnrest" ]; then rm -rf $(DESTDIR)$(plugindir)/clnrest; fi
 	[ -z "$(PLUGINS)" ] || $(INSTALL_PROGRAM) $(PLUGINS) $(DESTDIR)$(plugindir)
-	for PY in $(PY_PLUGINS); do DIR=`dirname $$PY`; DST=$(DESTDIR)$(plugindir)/`basename $$DIR`; if [ -d $$DST ]; then rm -rf $$DST; fi; $(INSTALL_PROGRAM) -d $$DIR; cp -a $$DIR $$DST ; done
 
 MAN1PAGES = $(filter %.1,$(MANPAGES))
 MAN5PAGES = $(filter %.5,$(MANPAGES))
 MAN7PAGES = $(filter %.7,$(MANPAGES))
 MAN8PAGES = $(filter %.8,$(MANPAGES))
-DOC_DATA = README.md LICENSE
+DOC_DATA = README.md doc/INSTALL.md doc/HACKING.md LICENSE
 
 install-data: installdirs $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA)
 	@$(NORMAL_INSTALL)
@@ -839,27 +707,6 @@ install-data: installdirs $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(
 
 install: install-program install-data
 
-# Non-artifacts that are needed for testing. These are added to the
-# testpack.tar, used to transfer things between builder and tester
-# phase. If you get a missing file/executable while testing on CI it
-# is likely missing from this variable.
-TESTBINS = \
-	$(CLN_PLUGIN_EXAMPLES) \
-	tests/plugins/test_libplugin \
-	tests/plugins/channeld_fakenet \
-	tests/plugins/test_selfdisable_after_getmanifest \
-	tools/hsmtool
-
-# The testpack is used in CI to transfer built artefacts between the
-# build and the test phase. This is necessary because the fixtures in
-# `tests/` explicitly use the binaries built in the current directory
-# rather than using `$PATH`, as that may pick up some other installed
-# version of `lightningd` leading to bogus results. We bundle up all
-# built artefacts here, and will unpack them on the tester (overlaying
-# on top of the checked out repo as if we had just built it in place).
-testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS)
-	tar -caf $@ $^
-
 uninstall:
 	@$(NORMAL_UNINSTALL)
 	@for f in $(BIN_PROGRAMS); do \
@@ -870,10 +717,6 @@ uninstall:
 	  $(ECHO) rm -f $(DESTDIR)$(plugindir)/`basename $$f`; \
 	  rm -f $(DESTDIR)$(plugindir)/`basename $$f`; \
 	done
-	@for f in $(PY_PLUGINS); do \
-	  $(ECHO) rm -rf $(DESTDIR)$(plugindir)/$$(basename $$(dirname $$f)); \
-	  rm -rf $(DESTDIR)$(plugindir)/$$(basename $$(dirname $$f)); \
-	done
 	@for f in $(PKGLIBEXEC_PROGRAMS); do \
 	  $(ECHO) rm -f $(DESTDIR)$(pkglibexecdir)/`basename $$f`; \
 	  rm -f $(DESTDIR)$(pkglibexecdir)/`basename $$f`; \
@@ -910,25 +753,18 @@ installcheck: all-programs
 	fi
 	@rm -rf testinstall || true
 
-version:
-	@echo ${VERSION}
-
 .PHONY: installdirs install-program install-data install uninstall \
-	installcheck ncc bin-tarball show-flags version
+	installcheck ncc bin-tarball show-flags
 
 # Make a tarball of opt/clightning/, optionally with label for distribution.
-ifneq ($(VERSION),)
 bin-tarball: clightning-$(VERSION)-$(DISTRO).tar.xz
 clightning-$(VERSION)-$(DISTRO).tar.xz: DESTDIR=$(shell pwd)/
 clightning-$(VERSION)-$(DISTRO).tar.xz: prefix=opt/clightning
 clightning-$(VERSION)-$(DISTRO).tar.xz: install
 	trap "rm -rf opt" 0; tar cvfa $@ opt/
-endif
 
 ccan-breakpoint.o: $(CCANDIR)/ccan/breakpoint/breakpoint.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
-ccan-base64.o: $(CCANDIR)/ccan/base64/base64.c
-	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-tal.o: $(CCANDIR)/ccan/tal/tal.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-tal-str.o: $(CCANDIR)/ccan/tal/str/str.c
@@ -1029,7 +865,3 @@ ccan-json_out.o: $(CCANDIR)/ccan/json_out/json_out.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-closefrom.o: $(CCANDIR)/ccan/closefrom/closefrom.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
-ccan-rune-rune.o: $(CCANDIR)/ccan/rune/rune.c
-	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
-ccan-rune-coding.o: $(CCANDIR)/ccan/rune/coding.c
-	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)

README.md

@@ -1,6 +1,6 @@
-# Core Lightning (CLN): A specification compliant Lightning Network implementation in C
+# c-lightning: A specification compliant Lightning Network implementation in C
 
-Core Lightning (previously c-lightning) is a lightweight, highly customizable and [standard compliant][std] implementation of the Lightning Network protocol.
+c-lightning is a lightweight, highly customizable and [standard compliant][std] implementation of the Lightning Network protocol.
 
 * [Getting Started](#getting-started)
     * [Installation](#installation)
@@ -15,38 +15,44 @@ Core Lightning (previously c-lightning) is a lightweight, highly customizable an
     * [Pruning](#pruning)
     * [HD wallet encryption](#hd-wallet-encryption)
 	* [Developers](#developers)
-* [Documentation](https://docs.corelightning.org/docs)
+* [Documentation](https://lightning.readthedocs.io/)
 
 ## Project Status
 
-[![Continuous Integration][actions-badge]][actions]
-[![Pull Requests Welcome][prs-badge]][prs]
-[![Documentation Status][docs-badge]][docs]
-[![BoL2][bol2-badge]][bol2]
-[![Telegram][telegram-badge]][telegram]
-[![Discord][discord-badge]][discord]
-[![Irc][IRC-badge]][IRC]
+[![Continuous Integration](https://github.com/ElementsProject/lightning/workflows/Continuous%20Integration/badge.svg)][actions]
+[![Pull Requests Welcome][prs]][prs-link]
+[![Irc][IRC]][IRC-link]
+[![Documentation Status](https://readthedocs.org/projects/lightning/badge/?version=docs)][docs]
 
 This implementation has been in production use on the Bitcoin mainnet since early 2018, with the launch of the [Blockstream Store][blockstream-store-blog].
-We recommend getting started by experimenting on `testnet` (`testnet4` or `regtest`), but the implementation is considered stable and can be safely used on mainnet.
-
-## Reach Out to Us
+We recommend getting started by experimenting on `testnet` (or `regtest`), but the implementation is considered stable and can be safely used on mainnet.
 
 Any help testing the implementation, reporting bugs, or helping with outstanding issues is very welcome.
-Don't hesitate to reach out to us on [Build-on-L2][bol2], or on the implementation-specific [mailing list][ml1], or on [CLN Discord][discord], or on [CLN Telegram][telegram], or on IRC at [dev][irc1]/[gen][irc2] channel.
+Don't hesitate to reach out to us on IRC at [#lightning-dev @ libera.chat][irc1], [#c-lightning @ libera.chat][irc2], or on the implementation-specific mailing list [c-lightning@lists.ozlabs.org][ml1], or on the Lightning Network-wide mailing list [lightning-dev@lists.linuxfoundation.org][ml2].
 
 ## Getting Started
 
-Core Lightning only works on Linux and macOS, and requires a locally (or remotely) running `bitcoind` (version 25.0 or above) that is fully caught up with the network you're running on, and relays transactions (ie with `blocksonly=0`).
+c-lightning only works on Linux and Mac OS, and requires a locally (or remotely) running `bitcoind` (version 0.16 or above) that is fully caught up with the network you're running on, and relays transactions (ie with `blocksonly=0`).
 Pruning (`prune=n` option in `bitcoin.conf`) is partially supported, see [here](#pruning) for more details.
 
 ### Installation
 
-There are 3 supported installation options:
+There are 4 supported installation options:
 
- - Installation of a pre-compiled binary from the [release page][releases] on GitHub.
+ - Installation from the [Ubuntu PPA][ppa].
+ - Installation of a pre-compiled binary from the [release page][releases] on Github.
  - Using one of the [provided docker images][dockerhub] on the Docker Hub.
- - Compiling the source code yourself as described in the [installation documentation](doc/getting-started/getting-started/installation.md).
+ - Compiling the source code yourself as described in the [installation documentation](doc/INSTALL.md).
+
+For the impatient here's the gist of it for Ubuntu:
+
+```bash
+sudo apt-get install -y software-properties-common
+sudo add-apt-repository -u ppa:lightningnetwork/ppa
+sudo apt-get install lightningd snapd
+sudo snap install bitcoin-core
+sudo ln -s /snap/bitcoin-core/current/bin/bitcoin{d,-cli} /usr/local/bin/
+```
 
 ### Starting `lightningd`
 
@@ -60,6 +66,13 @@ of the `startup_regtest.sh` file for details on how to use it.
 . contrib/startup_regtest.sh
 ```
 
+Note that your local nodeset will be much faster/more responsive if
+you've configured your node to expose the developer options, e.g.
+
+```bash
+./configure --enable-developer
+```
+
 #### Mainnet Option
 To test with real bitcoin,  you will need to have a local `bitcoind` node running:
 
@@ -78,11 +91,11 @@ You can start `lightningd` with the following command:
 lightningd --network=bitcoin --log-level=debug
 ```
 
-This creates a `.lightning/` subdirectory in your home directory: see `man -l doc/lightningd.8` (or https://docs.corelightning.org/docs) for more runtime options.
+This creates a `.lightning/` subdirectory in your home directory: see `man -l doc/lightningd.8` (or https://lightning.readthedocs.io/) for more runtime options.
 
 ### Using The JSON-RPC Interface
 
-Core Lightning exposes a [JSON-RPC 2.0][jsonrpcspec] interface over a Unix Domain socket; the `lightning-cli` tool can be used to access it, or there is a [python client library](contrib/pyln-client).
+c-lightning exposes a [JSON-RPC 2.0][jsonrpcspec] interface over a Unix Domain socket; the `lightning-cli` tool can be used to access it, or there is a [python client library](contrib/pyln-client).
 
 You can use `lightning-cli help` to print a table of RPC methods; `lightning-cli help <command>`
 will offer specific information on that command.
@@ -103,13 +116,18 @@ Once you've started for the first time, there's a script called
 `contrib/bootstrap-node.sh` which will connect you to other nodes on
 the lightning network.
 
-There are also numerous plugins available for Core Lightning which add
-capabilities: in particular there's a collection at: https://github.com/lightningd/plugins
+There are also numerous plugins available for c-lightning which add
+capabilities: in particular there's a collection at:
+
+	https://github.com/lightningd/plugins
+
+Including [helpme][helpme-github] which guides you through setting up
+your first channels and customizing your node.
 
 For a less reckless experience, you can encrypt the HD wallet seed:
  see [HD wallet encryption](#hd-wallet-encryption).
 
-You can also chat to other users at Discord [core-lightning][discord];
+You can also chat to other users at [#c-lightning @ libera.chat][irc2];
 we are always happy to help you get started!
 
 
@@ -125,11 +143,11 @@ lightning-cli newaddr
 
 `lightningd` will register the funds once the transaction is confirmed.
 
-Alternatively you can generate a taproot address should your source of funds support it:
+You may need to generate a p2sh-segwit address if the faucet does not support bech32:
 
 ```bash
-# Return a taproot address
-lightning-cli newaddr p2tr
+# Return a p2sh-segwit address
+lightning-cli newaddr p2sh-segwit
 ```
 
 Confirm `lightningd` got funds by:
@@ -165,7 +183,7 @@ lightning-cli invoice <amount> <label> <description>
 
 This returns some internal details, and a standard invoice string called `bolt11` (named after the [BOLT #11 lightning spec][BOLT11]).
 
-[BOLT11]: https://github.com/lightning/bolts/blob/master/11-payment-encoding.md
+[BOLT11]: https://github.com/lightningnetwork/lightning-rfc/blob/master/11-payment-encoding.md
 
 The sender can feed this `bolt11` string to the `decodepay` command to see what it is, and pay it simply using the `pay` command:
 
@@ -184,17 +202,15 @@ Command line options will always override the values in the configuration file.
 To use a configuration file, create a file named `config` within your top-level lightning directory or network subdirectory
 (eg. `~/.lightning/config` or `~/.lightning/bitcoin/config`).  See `man -l doc/lightningd-config.5`.
 
-A sample configuration file is available at `contrib/config-example`.
-
 ## Further information
 
 ### Pruning
 
-Core Lightning requires JSON-RPC access to a fully synchronized `bitcoind` in order to synchronize with the Bitcoin network.
+c-lightning requires JSON-RPC access to a fully synchronized `bitcoind` in order to synchronize with the Bitcoin network.
 Access to ZeroMQ is not required and `bitcoind` does not need to be run with `txindex` like other implementations.
 The lightning daemon will poll `bitcoind` for new blocks that it hasn't processed yet, thus synchronizing itself with `bitcoind`.
-If `bitcoind` prunes a block that Core Lightning has not processed yet, e.g., Core Lightning was not running for a prolonged period, then `bitcoind` will not be able to serve the missing blocks, hence Core Lightning will not be able to synchronize anymore and will be stuck.
-In order to avoid this situation you should be monitoring the gap between Core Lightning's blockheight using `lightning-cli getinfo` and `bitcoind`'s blockheight using `bitcoin-cli getblockchaininfo`.
+If `bitcoind` prunes a block that c-lightning has not processed yet, e.g., c-lightning was not running for a prolonged period, then `bitcoind` will not be able to serve the missing blocks, hence c-lightning will not be able to synchronize anymore and will be stuck.
+In order to avoid this situation you should be monitoring the gap between c-lightning's blockheight using `lightning-cli getinfo` and `bitcoind`'s blockheight using `bitcoin-cli getblockchaininfo`.
 If the two blockheights drift apart it might be necessary to intervene.
 
 ### HD wallet encryption
@@ -205,28 +221,25 @@ If you encrypt your `hsm_secret`, you will have to pass the `--encrypted-hsm` st
 
 ### Developers
 
-Developers wishing to contribute should start with the developer guide [here](doc/contribute-to-core-lightning/coding-style-guidelines.md).
+Developers wishing to contribute should start with the developer guide [here](doc/HACKING.md).
+You should also configure with `--enable-developer` to get additional checks and options.
 
 [blockstream-store-blog]: https://blockstream.com/2018/01/16/en-lightning-charge/
-[std]: https://github.com/lightning/bolts
-[prs-badge]: https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat
-[prs]: http://makeapullrequest.com
-[bol2-badge]: https://badgen.net/badge/BoL2/chat/blue
-[bol2]: https://community.corelightning.org
-[ml1]: https://lists.ozlabs.org/listinfo/c-lightning
-[discord-badge]: https://badgen.net/badge/Discord/chat/blue
-[discord]: https://discord.gg/mE9s4rc5un
-[telegram-badge]: https://badgen.net/badge/Telegram/chat/blue
-[telegram]: https://t.me/lightningd
-[IRC-badge]: https://img.shields.io/badge/IRC-chat-blue.svg
-[IRC]: https://web.libera.chat/#c-lightning
+[std]: https://github.com/lightningnetwork/lightning-rfc
+[travis-ci]: https://travis-ci.org/ElementsProject/lightning.svg?branch=master
+[travis-ci-link]: https://travis-ci.org/ElementsProject/lightning
+[prs]: https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat
+[prs-link]: http://makeapullrequest.com
+[IRC]: https://img.shields.io/badge/chat-on%20libera-brightgreen.svg
+[IRC-link]: https://web.libera.chat/#c-lightning
 [irc1]: https://web.libera.chat/#lightning-dev
 [irc2]: https://web.libera.chat/#c-lightning
-[docs-badge]: https://readthedocs.org/projects/lightning/badge/?version=docs
-[docs]: https://docs.corelightning.org/docs
+[ml1]: https://lists.ozlabs.org/listinfo/c-lightning
+[ml2]: https://lists.linuxfoundation.org/mailman/listinfo/lightning-dev
+[docs]: https://lightning.readthedocs.org
+[ppa]: https://launchpad.net/~lightningnetwork/+archive/ubuntu/ppa
 [releases]: https://github.com/ElementsProject/lightning/releases
 [dockerhub]: https://hub.docker.com/r/elementsproject/lightningd/
 [jsonrpcspec]: https://www.jsonrpc.org/specification
 [helpme-github]: https://github.com/lightningd/plugins/tree/master/helpme
-[actions-badge]: https://github.com/ElementsProject/lightning/workflows/Continuous%20Integration/badge.svg
 [actions]: https://github.com/ElementsProject/lightning/actions

SECURITY.md

@@ -1,35 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-We have a 3 month release cycle, and the last two versions are supported.
-
-## Reporting a Vulnerability
-
-To report security vulnerabilities, please send an email to one of the following addresses:
-- `rusty@rustcorp.com.au`
-- `security@blockstream.com`
-
-Note: These email addresses are exclusively for vulnerability reporting.
-
-For all other inquiries/communication, please refer to the [Reach Out to Us](https://github.com/ElementsProject/lightning?tab=readme-ov-file#reach-out-to-us) section in our README.
-
-## Signatures For Releases
-
-The following keys may be used to communicate sensitive information to
-developers, and to validate signatures on releases:
-
-| Name | Email | Fingerprint |
-|------|-------|-------------|
-| Blockstream Security Reporting | `security@blockstream.com` | 1176 542D A98E 71E1 3372  2EF7 4AC8 CC88 6844 A2D6 |
-| Rusty Russell | `rusty@rustcorp.com.au` | 15EE 8D6C AB0E 7F0C F999  BFCB D920 0E6C D1AD B8F1 |
-| Christian Decker | `decker@blockstream.com` | B731 AAC5 21B0 1385 9313  F674 A26D 6D9F E088 ED58 |
-| Lisa Neigut | `niftynei@gmail.com` | 30DE 693A E0DE 9E37 B3E7  EB6B BFF0 F678 10C1 EED1 |
-| Alex Myers | `alex@endothermic.dev` | 0437 4E42 789B BBA9 462E  4767 F3BF 63F2 7474 36AB |
-| Peter Neuroth | `pet.v.ne@gmail.com` | 653B 19F3 3DF7 EFF3 E9D1  C94C C3F2 1EE3 87FF 4CD2 |
-| Shahana Farooqui | `sfarooqui@blockstream.com` | FE13 58EB 7793 51DB 24E5  555A A327 573C 9758 9BF5 |
-| Blockstream CLN Release | `cln@blockstream.com` | 616C 52F9 9D06 12B2 A151  B107 4129 A994 AA7E 9852 |
-
-You can import a key by running the following command with that individual’s fingerprint: 
-`gpg --keyserver hkps://keys.openpgp.org --recv-keys "<fingerprint>"`. 
-Ensure that you put quotes around fingerprints containing spaces.

action.yml

@@ -0,0 +1,6 @@
+---
+name: 'Lightning CI'
+description: 'A preconfigured container with all c-lightning dependencies'
+runs:
+  using: 'docker'
+  image: 'contrib/Dockerfile.tester'

bitcoin/Makefile

@@ -41,9 +41,6 @@ BITCOIN_HEADERS := bitcoin/address.h		\
 # Bitcoin objects depends on bitcoin/ external/ and ccan
 $(BITCOIN_OBJS): $(CCAN_HEADERS) $(BITCOIN_HEADERS) $(EXTERNAL_HEADERS)
 
-ALL_C_HEADERS += $(BITCOIN_HEADERS)
-ALL_C_SOURCES += $(BITCOIN_SRC)
-
 check-makefile: check-bitcoin-makefile
 
 check-bitcoin-makefile:

bitcoin/base58.c

@@ -26,7 +26,7 @@ static char *to_base58(const tal_t *ctx, u8 version,
 				    total_length, BASE58_FLAG_CHECKSUM, &out)
 	    != WALLY_OK)
 		out = NULL;
-	tal_wally_end_onto(ctx, out, char);
+	tal_wally_end(tal_steal(ctx, out));
 
 	return out;
 }
@@ -47,14 +47,16 @@ static bool from_base58(u8 *version,
 			struct ripemd160 *rmd,
 			const char *base58, size_t base58_len)
 {
-	/* Initialize to avoid memcheck complaining if decoding a short value */
-	u8 buf[1 + sizeof(*rmd) + 4] = { 0 };
-	const size_t buflen = sizeof(buf);
-	const uint32_t flags = BASE58_FLAG_CHECKSUM;
+	u8 buf[1 + sizeof(*rmd) + 4];
+	/* Avoid memcheck complaining if decoding resulted in a short value */
+	size_t buflen = sizeof(buf);
+	memset(buf, 0, buflen);
+	char *terminated_base58 = tal_dup_arr(NULL, char, base58, base58_len, 1);
+	terminated_base58[base58_len] = '\0';
 
 	size_t written = 0;
-	int r = wally_base58_n_to_bytes(base58, base58_len, flags,
-					buf, buflen, &written);
+	int r = wally_base58_to_bytes(terminated_base58, BASE58_FLAG_CHECKSUM, buf, buflen, &written);
+	tal_free(terminated_base58);
 	if (r != WALLY_OK || written > buflen) {
 		return false;
 	}

bitcoin/block.c

@@ -1,11 +1,11 @@
 #include "config.h"
 #include <assert.h>
-#include <bitcoin/block.h>
 #include <bitcoin/chainparams.h>
 #include <bitcoin/tx.h>
 #include <ccan/mem/mem.h>
 #include <ccan/str/hex/hex.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
+#include <bitcoin/block.h>
 
 /* Sets *cursor to NULL and returns NULL when a pull fails. */
 static const u8 *pull(const u8 **cursor, size_t *max, void *copy, size_t n)
@@ -210,7 +210,7 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams,
 	b->tx = tal_arr(b, struct bitcoin_tx *, num);
 	b->txids = tal_arr(b, struct bitcoin_txid, num);
 	for (i = 0; i < num; i++) {
-		b->tx[i] = pull_bitcoin_tx_only(b->tx, &p, &len);
+		b->tx[i] = pull_bitcoin_tx(b->tx, &p, &len);
 		b->tx[i]->chainparams = chainparams;
 		bitcoin_txid(b->tx[i], &b->txids[i]);
 	}
@@ -237,14 +237,15 @@ static bool bitcoin_blkid_to_hex(const struct bitcoin_blkid *blockid,
 	return bitcoin_txid_to_hex(&fake_txid, hexstr, hexstr_len);
 }
 
-char *fmt_bitcoin_blkid(const tal_t *ctx,
-			const struct bitcoin_blkid *blkid)
+static char *fmt_bitcoin_blkid(const tal_t *ctx,
+			       const struct bitcoin_blkid *blkid)
 {
 	char *hexstr = tal_arr(ctx, char, hex_str_size(sizeof(*blkid)));
 
 	bitcoin_blkid_to_hex(blkid, hexstr, hex_str_size(sizeof(*blkid)));
 	return hexstr;
 }
+REGISTER_TYPE_TO_STRING(bitcoin_blkid, fmt_bitcoin_blkid);
 
 void fromwire_bitcoin_blkid(const u8 **cursor, size_t *max,
 			    struct bitcoin_blkid *blkid)

bitcoin/block.h

@@ -53,7 +53,4 @@ void fromwire_chainparams(const u8 **cursor, size_t *max,
 			  const struct chainparams **chainparams);
 void towire_chainparams(u8 **cursor, const struct chainparams *chainparams);
 
-char *fmt_bitcoin_blkid(const tal_t *ctx,
-			const struct bitcoin_blkid *blkid);
-
 #endif /* LIGHTNING_BITCOIN_BLOCK_H */

bitcoin/chainparams.c

@@ -1,5 +1,4 @@
 #include "config.h"
-#include <assert.h>
 #include <bitcoin/chainparams.h>
 #include <ccan/array_size/array_size.h>
 #include <ccan/tal/str/str.h>
@@ -28,8 +27,7 @@ static u8 liquid_regtest_fee_asset[] = {
 
 const struct chainparams networks[] = {
     {.network_name = "bitcoin",
-     .onchain_hrp = "bc",
-     .lightning_hrp = "bc",
+     .bip173_name = "bc",
      .bip70_name = "main",
      .genesis_blockhash = {{{.u.u8 = {0x6f, 0xe2, 0x8c, 0x0a, 0xb6, 0xf1, 0xb3,
 				      0x72, 0xc1, 0xa6, 0xa2, 0x46, 0xae, 0x63,
@@ -37,7 +35,6 @@ const struct chainparams networks[] = {
 				      0x5a, 0x08, 0x9c, 0x68, 0xd6, 0x19, 0x00,
 				      0x00, 0x00, 0x00, 0x00}}}},
      .rpc_port = 8332,
-     .ln_port = 9735,
      .cli = "bitcoin-cli",
      .cli_args = NULL,
      .cli_min_supported_version = 150000,
@@ -50,7 +47,6 @@ const struct chainparams networks[] = {
       */
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      /* "Lightning Charge Powers Developers & Blockstream Store" */
      .when_lightning_became_cool = 504500,
      .p2pkh_version = 0,
@@ -61,8 +57,7 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_MAIN_PRIVATE},
      .is_elements = false},
     {.network_name = "regtest",
-     .onchain_hrp = "bcrt",
-     .lightning_hrp = "bcrt",
+     .bip173_name = "bcrt",
      .bip70_name = "regtest",
      .genesis_blockhash = {{{.u.u8 = {0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b,
 				      0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb,
@@ -70,14 +65,12 @@ const struct chainparams networks[] = {
 				      0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c,
 				      0xf1, 0x88, 0x91, 0x0f}}}},
      .rpc_port = 18443,
-     .ln_port = 19846,
      .cli = "bitcoin-cli",
      .cli_args = "-regtest",
      .cli_min_supported_version = 150000,
      .dust_limit = { 546 },
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 111,
      .p2sh_version = 196,
@@ -87,8 +80,7 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
      .is_elements = false},
     {.network_name = "signet",
-     .onchain_hrp = "tb",
-     .lightning_hrp = "tbs",
+     .bip173_name = "tbs",
      .bip70_name = "signet",
      // 00000008819873e925422c1ff0f99f7cc9bbb232af63a077a480a3633bee1ef6
      .genesis_blockhash = {{{.u.u8 = {0xf6, 0x1e, 0xee, 0x3b, 0x63, 0xa3, 0x80,
@@ -97,14 +89,12 @@ const struct chainparams networks[] = {
 				      0x2c, 0x42, 0x25, 0xe9, 0x73, 0x98, 0x81,
 				      0x08, 0x00, 0x00, 0x00}}}},
      .rpc_port = 38332,
-     .ln_port = 39735,
      .cli = "bitcoin-cli",
      .cli_args = "-signet",
      .cli_min_supported_version = 150000,
      .dust_limit = { 546 },
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 111,
      .p2sh_version = 196,
@@ -113,8 +103,7 @@ const struct chainparams networks[] = {
      .is_elements = false,
     },
     {.network_name = "testnet",
-     .onchain_hrp = "tb",
-     .lightning_hrp = "tb",
+     .bip173_name = "tb",
      .bip70_name = "test",
      .genesis_blockhash = {{{.u.u8 = {0x43, 0x49, 0x7f, 0xd7, 0xf8, 0x26, 0x95,
 				      0x71, 0x08, 0xf4, 0xa3, 0x0f, 0xd9, 0xce,
@@ -122,40 +111,12 @@ const struct chainparams networks[] = {
 				      0xe9, 0x0e, 0xad, 0x01, 0xea, 0x33, 0x09,
 				      0x00, 0x00, 0x00, 0x00}}}},
      .rpc_port = 18332,
-     .ln_port = 19735,
      .cli = "bitcoin-cli",
      .cli_args = "-testnet",
      .cli_min_supported_version = 150000,
      .dust_limit = { 546 },
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
-     .p2pkh_version = 111,
-     .p2sh_version = 196,
-     .testnet = true,
-     .fee_asset_tag = NULL,
-     .bip32_key_version = {.bip32_pubkey_version = BIP32_VER_TEST_PUBLIC,
-			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
-     .is_elements = false},
-    {.network_name = "testnet4",
-     .onchain_hrp = "tb",
-     .lightning_hrp = "tb",
-     .bip70_name = "testnet4",
-     // 00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043
-     .genesis_blockhash = {{{.u.u8 = {0x43, 0xf0, 0x8b, 0xda, 0xb0, 0x50, 0xe3,
-				      0x5b, 0x56, 0x7c, 0x86, 0x4b, 0x91, 0xf4,
-				      0x7f, 0x50, 0xae, 0x72, 0x5a, 0xe2, 0xde,
-				      0x53, 0xbc, 0xfb, 0xba, 0xf2, 0x84, 0xda,
-				      0x00, 0x00, 0x00, 0x00}}}},
-     .rpc_port = 48332,
-     .ln_port = 49735,
-     .cli = "bitcoin-cli",
-     .cli_args = "-testnet4",
-     .cli_min_supported_version = 150000,
-     .dust_limit = { 546 },
-     .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
-     .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .p2pkh_version = 111,
      .p2sh_version = 196,
      .testnet = true,
@@ -164,8 +125,7 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
      .is_elements = false},
     {.network_name = "litecoin",
-     .onchain_hrp = "ltc",
-     .lightning_hrp = "ltc",
+     .bip173_name = "ltc",
      .bip70_name = "main",
      .genesis_blockhash = {{{.u.u8 = {0xe2, 0xbf, 0x04, 0x7e, 0x7e, 0x5a, 0x19,
 				      0x1a, 0xa4, 0xef, 0x34, 0xd3, 0x14, 0x97,
@@ -173,14 +133,12 @@ const struct chainparams networks[] = {
 				      0x1e, 0xda, 0xba, 0x59, 0x40, 0xfd, 0x1f,
 				      0xe3, 0x65, 0xa7, 0x12}}}},
      .rpc_port = 9332,
-     .ln_port = 9735,
      .cli = "litecoin-cli",
      .cli_args = NULL,
      .cli_min_supported_version = 150000,
      .dust_limit = { 100000 },
      .max_funding = AMOUNT_SAT_INIT(60 * ((1 << 24) - 1)),
      .max_payment = AMOUNT_MSAT_INIT(60 * 0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1320000,
      .p2pkh_version = 48,
      .p2sh_version = 50,
@@ -190,8 +148,7 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_MAIN_PRIVATE},
      .is_elements = false},
     {.network_name = "litecoin-testnet",
-     .onchain_hrp = "tltc",
-     .lightning_hrp = "tltc",
+     .bip173_name = "tltc",
      .bip70_name = "test",
      .genesis_blockhash = {{{.u.u8 = {0xa0, 0x29, 0x3e, 0x4e, 0xeb, 0x3d, 0xa6,
 				      0xe6, 0xf5, 0x6f, 0x81, 0xed, 0x59, 0x5f,
@@ -199,14 +156,12 @@ const struct chainparams networks[] = {
 				      0x13, 0xee, 0xfd, 0xd9, 0x51, 0x28, 0x4b,
 				      0x5a, 0x62, 0x66, 0x49}}}},
      .rpc_port = 19332,
-     .ln_port = 9735,
      .cli = "litecoin-cli",
      .cli_args = "-testnet",
      .cli_min_supported_version = 150000,
      .dust_limit = { 100000 },
      .max_funding = AMOUNT_SAT_INIT(60 * ((1 << 24) - 1)),
      .max_payment = AMOUNT_MSAT_INIT(60 * 0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 111,
      .p2sh_version = 58,
@@ -216,8 +171,7 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
      .is_elements = false},
     {.network_name = "liquid-regtest",
-     .onchain_hrp = "ert",
-     .lightning_hrp = "ert",
+     .bip173_name = "ert",
      .bip70_name = "liquid-regtest",
      .genesis_blockhash = {{{.u.u8 = {0x9f, 0x87, 0xeb, 0x58, 0x0b, 0x9e, 0x5f,
 				      0x11, 0xdc, 0x21, 0x1e, 0x9f, 0xb6, 0x6a,
@@ -225,13 +179,11 @@ const struct chainparams networks[] = {
 				      0xfe, 0x14, 0x68, 0x01, 0x16, 0x23, 0x93,
 				      0x36, 0x42, 0x86, 0xc6}}}},
      .rpc_port = 19332,
-     .ln_port = 20735,
      .cli = "elements-cli",
      .cli_args = "-chain=liquid-regtest",
      .dust_limit = {546},
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 91,
      .p2sh_version = 75,
@@ -241,8 +193,7 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
      .is_elements = true},
     {.network_name = "liquid",
-     .onchain_hrp = "ex",
-     .lightning_hrp = "ex",
+     .bip173_name = "ex",
      .bip70_name = "liquidv1",
      .genesis_blockhash = {{{.u.u8 = {0x14, 0x66, 0x27, 0x58, 0x36, 0x22, 0x0d,
 				      0xb2, 0x94, 0x4c, 0xa0, 0x59, 0xa3, 0xa1,
@@ -250,13 +201,11 @@ const struct chainparams networks[] = {
 				      0x68, 0x8d, 0x2c, 0x37, 0x92, 0x96, 0x88,
 				      0x8a, 0x20, 0x60, 0x03}}}},
      .rpc_port = 7041,
-     .ln_port = 9735,
      .cli = "elements-cli",
      .cli_args = "-chain=liquidv1",
      .dust_limit = {546},
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 57,
      .p2sh_version = 39,
@@ -287,10 +236,10 @@ const struct chainparams *chainparams_by_chainhash(const struct bitcoin_blkid *c
 	return NULL;
 }
 
-const struct chainparams *chainparams_by_lightning_hrp(const char *lightning_hrp)
+const struct chainparams *chainparams_by_bip173(const char *bip173_name)
 {
 	for (size_t i = 0; i < ARRAY_SIZE(networks); i++) {
-		if (streq(lightning_hrp, networks[i].lightning_hrp)) {
+		if (streq(bip173_name, networks[i].bip173_name)) {
 			return &networks[i];
 		}
 	}
@@ -305,8 +254,3 @@ const char *chainparams_get_network_names(const tal_t *ctx)
     return networks_string;
 }
 
-int chainparams_get_ln_port(const struct chainparams *params)
-{
-	assert(params);
-	return params->ln_port;
-}

bitcoin/chainparams.h

@@ -10,28 +10,12 @@
 
 struct chainparams {
 	const char *network_name;
-	/* Unfortunately starting with signet, we now have diverging
-	 * conventions for the "BIP173" Human Readable Part (HRP).
-	 * On onchain signet, the HRP is `tb` , but on Lightning
-	 * signet the HRP is `tbs`.
-	 */
-	const char *onchain_hrp;
-	const char *lightning_hrp;
+	const char *bip173_name;
 	/*'bip70_name' is corresponding to the 'chain' field of
 	 * the API 'getblockchaininfo' */
 	const char *bip70_name;
 	const struct bitcoin_blkid genesis_blockhash;
 	const int rpc_port;
-	/**
-	 * BOLT 1:
-	 *
-	 * The default TCP port depends on the network used. The most common networks are:
-	 *
-	 * - Bitcoin mainet with port number 9735 or the corresponding hexadecimal `0x2607`;
-	 * - Bitcoin testnet with port number 19735 (`0x4D17`);
-	 * - Bitcoin signet with port number 39735 (`0x9B37`).
-	 */
-	const int ln_port;
 	const char *cli;
 	const char *cli_args;
 	/* The min numeric version of cli supported */
@@ -39,8 +23,6 @@ struct chainparams {
 	const struct amount_sat dust_limit;
 	const struct amount_sat max_funding;
 	const struct amount_msat max_payment;
-	/* Total coins in network */
-	const struct amount_sat max_supply;
 	const u32 when_lightning_became_cool;
 	const u8 p2pkh_version;
 	const u8 p2sh_version;
@@ -64,7 +46,7 @@ const struct chainparams *chainparams_for_network(const char *network_name);
  *
  * This lets us decode BOLT11 addresses.
  */
-const struct chainparams *chainparams_by_lightning_hrp(const char *lightning_hrp);
+const struct chainparams *chainparams_by_bip173(const char *bip173_name);
 
 /**
  * chainparams_by_chainhash - Helper to get a network by its genesis blockhash
@@ -76,9 +58,4 @@ const struct chainparams *chainparams_by_chainhash(const struct bitcoin_blkid *c
  */
 const char *chainparams_get_network_names(const tal_t *ctx);
 
-/**
- * chainparams_get_ln_port - Return the lightning network default port by
- * network if the chainparams is initialized, otherwise 9735 as mock port
- */
-int chainparams_get_ln_port(const struct chainparams *params);
 #endif /* LIGHTNING_BITCOIN_CHAINPARAMS_H */

bitcoin/feerate.c

@@ -1,12 +1,8 @@
 #include "config.h"
-#include <assert.h>
 #include <bitcoin/feerate.h>
 
 u32 feerate_from_style(u32 feerate, enum feerate_style style)
 {
-	/* Make sure it's called somewhere! */
-	assert(feerate_floor_check() == FEERATE_FLOOR);
-
 	switch (style) {
 	case FEERATE_PER_KSIPA:
 		return feerate;

bitcoin/feerate.h

@@ -39,7 +39,7 @@ enum feerate_style {
 	FEERATE_PER_KBYTE
 };
 
-static inline u32 feerate_floor_check(void)
+static inline u32 feerate_floor(void)
 {
 	/* Assert that bitcoind will see this as above minRelayTxFee */
 	BUILD_ASSERT(FEERATE_BITCOIND_SEES(FEERATE_FLOOR, MINIMUM_TX_WEIGHT)

bitcoin/locktime.c

@@ -2,6 +2,7 @@
 #include <assert.h>
 #include <bitcoin/locktime.h>
 #include <ccan/tal/str/str.h>
+#include <common/type_to_string.h>
 
 #define SECONDS_POINT 500000000
 

bitcoin/preimage.c

@@ -1,6 +1,5 @@
 #include "config.h"
 #include <bitcoin/preimage.h>
-#include <common/utils.h>
 #include <wire/wire.h>
 
 void fromwire_preimage(const u8 **cursor, size_t *max, struct preimage *preimage)
@@ -13,7 +12,4 @@ void towire_preimage(u8 **pptr, const struct preimage *preimage)
 	towire(pptr, preimage, sizeof(*preimage));
 }
 
-char *fmt_preimage(const tal_t *ctx, const struct preimage *preimage)
-{
-	return tal_hexstr(ctx, preimage, sizeof(*preimage));
-}
+

bitcoin/preimage.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/short_types/short_types.h>
 #include <ccan/structeq/structeq.h>
-#include <ccan/tal/tal.h>
 
 struct preimage {
 	u8 r[32];
@@ -14,5 +13,4 @@ STRUCTEQ_DEF(preimage, 0, r);
 void fromwire_preimage(const u8 **cursor, size_t *max, struct preimage *preimage);
 void towire_preimage(u8 **pptr, const struct preimage *preimage);
 
-char *fmt_preimage(const tal_t *ctx, const struct preimage *preimage);
 #endif /* LIGHTNING_BITCOIN_PREIMAGE_H */

bitcoin/privkey.c

@@ -1,10 +1,10 @@
 #include "config.h"
 #include <bitcoin/privkey.h>
 #include <ccan/str/hex/hex.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
-char *fmt_privkey(const tal_t *ctx, const struct privkey *secret)
+static char *privkey_to_hexstr(const tal_t *ctx, const struct privkey *secret)
 {
 	/* Bitcoin appends "01" to indicate the pubkey is compressed. */
 	char *str = tal_arr(ctx, char, hex_str_size(sizeof(*secret) + 1));
@@ -12,11 +12,8 @@ char *fmt_privkey(const tal_t *ctx, const struct privkey *secret)
 	strcat(str, "01");
 	return str;
 }
-
-char *fmt_secret(const tal_t *ctx, const struct secret *secret)
-{
-	return tal_hexstr(ctx, secret, sizeof(*secret));
-}
+REGISTER_TYPE_TO_STRING(privkey, privkey_to_hexstr);
+REGISTER_TYPE_TO_HEXSTR(secret);
 
 bool secret_eq_consttime(const struct secret *a, const struct secret *b)
 {

bitcoin/privkey.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/short_types/short_types.h>
 #include <ccan/structeq/structeq.h>
-#include <ccan/tal/tal.h>
 
 #define PRIVKEY_LEN 32
 
@@ -26,7 +25,4 @@ void fromwire_privkey(const u8 **cursor, size_t *max, struct privkey *privkey);
 void towire_privkey(u8 **pptr, const struct privkey *privkey);
 void towire_secret(u8 **pptr, const struct secret *secret);
 
-char *fmt_privkey(const tal_t *ctx, const struct privkey *privkey);
-char *fmt_secret(const tal_t *ctx, const struct secret *secret);
-
 #endif /* LIGHTNING_BITCOIN_PRIVKEY_H */

bitcoin/psbt.h

@@ -17,11 +17,6 @@ struct bitcoin_signature;
 struct bitcoin_txid;
 struct pubkey;
 
-
-/* Utility we need for psbt stuffs;
- * add the varint onto the given array */
-void add_varint(u8 **arr, size_t val);
-
 /**
  * create_psbt - Create a new psbt object
  *
@@ -34,7 +29,7 @@ struct wally_psbt *create_psbt(const tal_t *ctx, size_t num_inputs, size_t num_o
 
 /*
  * new_psbt - Create a PSBT, using the passed in tx
- * 	      as the locktime/inputs/output psbt fields
+ * 	      as the global_tx
  *
  * @ctx - allocation context
  * @wtx - global_tx starter kit
@@ -48,29 +43,7 @@ struct wally_psbt *new_psbt(const tal_t *ctx,
  * @ctx - allocation context
  * @psbt - psbt to be cloned
  */
-struct wally_psbt *clone_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
-
-/**
- * combine_psbt - Combine two PSBT into a cloned copy
- *
- * @ctx - allocation context
- * @psbt0 - one psbt
- * @psbt1 - other psbt
- */
-struct wally_psbt *combine_psbt(const tal_t *ctx,
-				const struct wally_psbt *psbt0,
-				const struct wally_psbt *psbt1);
-
-/**
- * audit_psbt - Audit the memory structure of the PSBT.
- *
- * This checks all known memory allocations in the PSBT and asserts that they
- * are all allocated with 'ctx' being it's parent.
- *
- * ctx - the ctx all memory *should* be attached to
- * psbt - the PSBT to audit.
- * */
-void audit_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
+struct wally_psbt *clone_psbt(const tal_t *ctx, struct wally_psbt *psbt);
 
 /**
  * psbt_is_finalized - Check if tx is ready to be extracted
@@ -119,7 +92,7 @@ bool psbt_finalize(struct wally_psbt *psbt);
  */
 struct wally_tx *psbt_final_tx(const tal_t *ctx, const struct wally_psbt *psbt);
 
-/* psbt_make_key - Create a new, proprietary Core Lightning key
+/* psbt_make_key - Create a new, proprietary c-lightning key
  *
  * @ctx - allocation context
  * @key_subtype - type for this key
@@ -130,7 +103,7 @@ struct wally_tx *psbt_final_tx(const tal_t *ctx, const struct wally_psbt *psbt);
 u8 *psbt_make_key(const tal_t *ctx, u8 key_subtype, const u8 *key_data);
 
 struct wally_psbt_input *psbt_add_input(struct wally_psbt *psbt,
-					const struct wally_tx_input *input,
+					struct wally_tx_input *input,
 					size_t insert_at);
 
 /* One stop shop for adding an input + metadata to a PSBT */
@@ -149,9 +122,6 @@ void psbt_input_set_wit_utxo(struct wally_psbt *psbt, size_t in,
 void psbt_input_set_utxo(struct wally_psbt *psbt, size_t in,
 			 const struct wally_tx *prev_tx);
 
-void psbt_input_set_outpoint(struct wally_psbt *psbt, size_t in,
-			     struct bitcoin_outpoint outpoint);
-
 /* psbt_elements_input_set_asset - Set the asset/value fields for an
  * 				   Elements PSBT (PSET, technically */
 void psbt_elements_input_set_asset(struct wally_psbt *psbt, size_t in,
@@ -183,28 +153,12 @@ void psbt_rm_output(struct wally_psbt *psbt,
 		    size_t remove_at);
 
 void psbt_input_add_pubkey(struct wally_psbt *psbt, size_t in,
-			   const struct pubkey *pubkey, bool is_taproot);
+			   const struct pubkey *pubkey);
 
 WARN_UNUSED_RESULT bool psbt_input_set_signature(struct wally_psbt *psbt, size_t in,
 						 const struct pubkey *pubkey,
 						 const struct bitcoin_signature *sig);
 
-/* Returns false on error. On success, *signature_found is set to true if the
- * input has a signature present for `pubkey` and false if if one was not found.
- * Only signature presence is checked, it is not validated. */
-WARN_UNUSED_RESULT bool psbt_input_have_signature(const struct wally_psbt *psbt,
-						  size_t in,
-						  const struct pubkey *pubkey,
-						  bool *signature_found);
-
-/* Returns false on error. On success *sig is set to the signature otherwise
- * *sig is set to NULL. */
-WARN_UNUSED_RESULT bool psbt_input_get_ecdsa_sig(const tal_t *ctx,
-						 const struct wally_psbt *psbt,
-						 size_t in,
-						 const struct pubkey *pubkey,
-						 struct bitcoin_signature **sig);
-
 void psbt_input_set_witscript(struct wally_psbt *psbt, size_t in, const u8 *wscript);
 
 /* psbt_input_set_unknown - Set the given Key-Value in the psbt's input keymap
@@ -231,19 +185,6 @@ void *psbt_get_lightning(const struct wally_map *map,
 			 const u8 proprietary_type,
 			 size_t *val_len);
 
-/* psbt_set_lightning - Set a propreitary lightning value on the given map
- *
- * @map - map of unknowns to set the value
- * @proprietary_type - type no. to set
- * @value - the value to be set
- * @val_len - length of value
- */
-void psbt_set_lightning(const tal_t *ctx,
-			struct wally_map *map,
-			const u8 proprietary_type,
-			const void *value,
-			size_t val_len);
-
 /* psbt_output_set_unknown - Set the given Key-Value in the psbt's output keymap
  *
  * @ctx - tal context for allocations
@@ -265,10 +206,6 @@ void psbt_output_set_unknown(const tal_t *ctx,
 struct amount_sat psbt_input_get_amount(const struct wally_psbt *psbt,
 					size_t in);
 
-/* psbt_input_get_weight - Calculate the tx weight for input index `in` */
-size_t psbt_input_get_weight(const struct wally_psbt *psbt,
-			     size_t in);
-
 /* psbt_output_get_amount - Returns the value of this output
  *
  * @psbt - psbt
@@ -277,10 +214,6 @@ size_t psbt_input_get_weight(const struct wally_psbt *psbt,
 struct amount_sat psbt_output_get_amount(const struct wally_psbt *psbt,
 					 size_t out);
 
-/* psbt_output_get_weight - Calculate the tx weight for output index `outnum` */
-size_t psbt_output_get_weight(const struct wally_psbt *psbt,
-			      size_t outnum);
-
 /* psbt_compute_fee - Returns value of fee for PSBT
  *
  * @psbt -psbt
@@ -295,40 +228,12 @@ struct amount_sat psbt_compute_fee(const struct wally_psbt *psbt);
 bool psbt_has_input(const struct wally_psbt *psbt,
 		    const struct bitcoin_outpoint *outpoint);
 
-/* wally_psbt_input_spends - Returns true if PSBT input spends given outpoint
- *
- * @input - psbt input
- * @outpoint - outpoint
- */
-bool wally_psbt_input_spends(const struct wally_psbt_input *input,
-               const struct bitcoin_outpoint *outpoint);
-
-void wally_psbt_input_get_outpoint(const struct wally_psbt_input *in,
-                 struct bitcoin_outpoint *outpoint);
-
-const u8 *wally_psbt_output_get_script(const tal_t *ctx,
-                     const struct wally_psbt_output *output);
-
-void wally_psbt_input_get_txid(const struct wally_psbt_input *in,
-                 struct bitcoin_txid *txid);
-
-struct amount_asset
-wally_psbt_output_get_amount(const struct wally_psbt_output *output);
-
-/* psbt_set_version - Returns false if there was any issue with the PSBT.
- * Returns true if it was a well-formed PSET and treats it as a no-op
- */
-bool psbt_set_version(struct wally_psbt *psbt, u32 version);
-
-bool elements_psbt_output_is_fee(const struct wally_psbt *psbt, size_t outnum);
-
 struct wally_psbt *psbt_from_b64(const tal_t *ctx,
 				 const char *b64,
 				 size_t b64len);
-char *fmt_wally_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
+char *psbt_to_b64(const tal_t *ctx, const struct wally_psbt *psbt);
 const u8 *psbt_get_bytes(const tal_t *ctx, const struct wally_psbt *psbt,
 			 size_t *bytes_written);
-bool validate_psbt(const struct wally_psbt *psbt);
 struct wally_psbt *psbt_from_bytes(const tal_t *ctx, const u8 *bytes,
 				   size_t byte_len);
 void towire_wally_psbt(u8 **pptr, const struct wally_psbt *psbt);

bitcoin/pubkey.c

@@ -4,7 +4,7 @@
 #include <bitcoin/pubkey.h>
 #include <ccan/mem/mem.h>
 #include <ccan/str/hex/hex.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
 #ifndef SUPERVERBOSE
@@ -62,15 +62,16 @@ bool pubkey_from_hexstr(const char *derstr, size_t slen, struct pubkey *key)
 	return pubkey_from_der(der, dlen, key);
 }
 
-char *fmt_pubkey(const tal_t *ctx, const struct pubkey *key)
+char *pubkey_to_hexstr(const tal_t *ctx, const struct pubkey *key)
 {
 	unsigned char der[PUBKEY_CMPR_LEN];
 
 	pubkey_to_der(der, key);
 	return tal_hexstr(ctx, der, sizeof(der));
 }
+REGISTER_TYPE_TO_STRING(pubkey, pubkey_to_hexstr);
 
-char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key)
+static char *secp256k1_pubkey_to_hexstr(const tal_t *ctx, const secp256k1_pubkey *key)
 {
 	unsigned char der[PUBKEY_CMPR_LEN];
 	size_t outlen = sizeof(der);
@@ -80,10 +81,11 @@ char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key)
 	assert(outlen == sizeof(der));
 	return tal_hexstr(ctx, der, sizeof(der));
 }
+REGISTER_TYPE_TO_STRING(secp256k1_pubkey, secp256k1_pubkey_to_hexstr);
 
 int pubkey_cmp(const struct pubkey *a, const struct pubkey *b)
 {
-	u8 keya[PUBKEY_CMPR_LEN], keyb[PUBKEY_CMPR_LEN];
+	u8 keya[33], keyb[33];
 	pubkey_to_der(keya, a);
 	pubkey_to_der(keyb, b);
 	return memcmp(keya, keyb, sizeof(keya));
@@ -123,3 +125,37 @@ void towire_pubkey(u8 **pptr, const struct pubkey *pubkey)
 
 	towire(pptr, output, outputlen);
 }
+
+void fromwire_point32(const u8 **cursor, size_t *max, struct point32 *point32)
+{
+	u8 raw[32];
+
+	if (!fromwire(cursor, max, raw, sizeof(raw)))
+		return;
+
+	if (secp256k1_xonly_pubkey_parse(secp256k1_ctx,
+					 &point32->pubkey,
+					 raw) != 1) {
+		SUPERVERBOSE("not a valid point");
+		fromwire_fail(cursor, max);
+	}
+}
+
+void towire_point32(u8 **pptr, const struct point32 *point32)
+{
+	u8 output[32];
+
+	secp256k1_xonly_pubkey_serialize(secp256k1_ctx, output,
+					 &point32->pubkey);
+	towire(pptr, output, sizeof(output));
+}
+
+static char *point32_to_hexstr(const tal_t *ctx, const struct point32 *point32)
+{
+	u8 output[32];
+
+	secp256k1_xonly_pubkey_serialize(secp256k1_ctx, output,
+					 &point32->pubkey);
+	return tal_hexstr(ctx, output, sizeof(output));
+}
+REGISTER_TYPE_TO_STRING(point32, point32_to_hexstr);

bitcoin/pubkey.h

@@ -19,12 +19,18 @@ struct pubkey {
 /* Define pubkey_eq (no padding) */
 STRUCTEQ_DEF(pubkey, 0, pubkey.data);
 
+struct point32 {
+	/* Unpacked pubkey (as used by libsecp256k1 internally) */
+	secp256k1_xonly_pubkey pubkey;
+};
+/* Define pubkey_eq (no padding) */
+STRUCTEQ_DEF(point32, 0, pubkey.data);
+
 /* Convert from hex string of DER (scriptPubKey from validateaddress) */
 bool pubkey_from_hexstr(const char *derstr, size_t derlen, struct pubkey *key);
 
 /* Convert from hex string of DER (scriptPubKey from validateaddress) */
-char *fmt_pubkey(const tal_t *ctx, const struct pubkey *key);
-char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key);
+char *pubkey_to_hexstr(const tal_t *ctx, const struct pubkey *key);
 
 /* Point from secret */
 bool pubkey_from_secret(const struct secret *secret, struct pubkey *key);
@@ -57,4 +63,13 @@ void pubkey_to_hash160(const struct pubkey *pk, struct ripemd160 *hash);
 void towire_pubkey(u8 **pptr, const struct pubkey *pubkey);
 void fromwire_pubkey(const u8 **cursor, size_t *max, struct pubkey *pubkey);
 
+/* FIXME: Old spec uses pubkey32 */
+#define pubkey32 point32
+#define towire_pubkey32 towire_point32
+#define fromwire_pubkey32 fromwire_point32
+
+/* marshal/unmarshal functions */
+void towire_point32(u8 **pptr, const struct point32 *pubkey);
+void fromwire_point32(const u8 **cursor, size_t *max, struct point32 *pubkey);
+
 #endif /* LIGHTNING_BITCOIN_PUBKEY_H */

bitcoin/script.c

@@ -310,77 +310,14 @@ u8 *scriptpubkey_witness_raw(const tal_t *ctx, u8 version,
 	return script;
 }
 
-u8 *scriptpubkey_raw_p2tr(const tal_t *ctx, const struct pubkey *output_pubkey)
-{
-	int ok;
-	secp256k1_xonly_pubkey x_key;
-	unsigned char x_key_bytes[32];
-	u8 *script = tal_arr(ctx, u8, 0);
-
-	add_op(&script, OP_1);
-
-	ok = secp256k1_xonly_pubkey_from_pubkey(secp256k1_ctx,
-		&x_key,
-		/* pk_parity */ NULL,
-		&(output_pubkey->pubkey));
-	assert(ok);
-
-	ok = secp256k1_xonly_pubkey_serialize(secp256k1_ctx,
-		x_key_bytes,
-		&x_key);
-	assert(ok);
-
-	script_push_bytes(&script, x_key_bytes, sizeof(x_key_bytes));
-	assert(tal_count(script) == BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
-	return script;
-}
-
-u8 *scriptpubkey_raw_p2tr_derkey(const tal_t *ctx, const u8 output_der[33])
-{
-	struct pubkey tr_key;
-	if (!pubkey_from_der(output_der, 33, &tr_key)) {
-		abort();
-	}
-	return scriptpubkey_raw_p2tr(ctx, &tr_key);
-}
-
-u8 *scriptpubkey_p2tr(const tal_t *ctx, const struct pubkey *inner_pubkey)
-{
-	unsigned char key_bytes[33];
-	unsigned char tweaked_key_bytes[33];
-	size_t out_len = sizeof(key_bytes);
-	u8 *script = tal_arr(ctx, u8, 0);
-
-	add_op(&script, OP_1);
-
-	secp256k1_ec_pubkey_serialize(secp256k1_ctx, key_bytes, &out_len, &inner_pubkey->pubkey, SECP256K1_EC_COMPRESSED);
-	/* Only commit to inner pubkey in tweak */
-	if (wally_ec_public_key_bip341_tweak(key_bytes, 33, /* merkle_root*/ NULL, 0, 0 /* flags */, tweaked_key_bytes, sizeof(tweaked_key_bytes)) != WALLY_OK)
-		abort();
-
-	/* Cut off the first byte from the serialized compressed key */
-	script_push_bytes(&script, tweaked_key_bytes + 1, sizeof(tweaked_key_bytes) - 1);
-	assert(tal_count(script) == BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
-	return script;
-}
-
-u8 *scriptpubkey_p2tr_derkey(const tal_t *ctx, const u8 inner_der[33])
-{
-	struct pubkey tr_key;
-	if (!pubkey_from_der(inner_der, 33, &tr_key)) {
-		abort();
-	}
-	return scriptpubkey_p2tr(ctx, &tr_key);
-}
-
 /* BOLT #3:
  *
  * #### `to_remote` Output
  *
- * If `option_anchors` applies to the commitment
+ * If `option_anchor_outputs` applies to the commitment
  * transaction, the `to_remote` output is encumbered by a one
  * block csv lock.
- *    <remotepubkey> OP_CHECKSIGVERIFY 1 OP_CHECKSEQUENCEVERIFY
+ *    <remote_pubkey> OP_CHECKSIGVERIFY 1 OP_CHECKSEQUENCEVERIFY
  */
 /* BOLT- #3
  * ##### Leased channel (`option_will_fund`)
@@ -392,9 +329,9 @@ u8 *scriptpubkey_p2tr_derkey(const tal_t *ctx, const u8 inner_der[33])
  * <remote_pubkey> OP_CHECKSIGVERIFY MAX(1, lease_end - blockheight) OP_CHECKSEQUENCEVERIFY
  */
 
-u8 *bitcoin_wscript_to_remote_anchored(const tal_t *ctx,
-				       const struct pubkey *remote_key,
-				       u32 csv_lock)
+u8 *anchor_to_remote_redeem(const tal_t *ctx,
+			    const struct pubkey *remote_key,
+			    u32 csv_lock)
 {
 	u8 *script = tal_arr(ctx, u8, 0);
 	add_push_key(&script, remote_key);
@@ -402,11 +339,11 @@ u8 *bitcoin_wscript_to_remote_anchored(const tal_t *ctx,
 	add_number(&script, csv_lock);
 	add_op(&script, OP_CHECKSEQUENCEVERIFY);
 
-	assert(is_to_remote_anchored_witness_script(script, tal_bytelen(script)));
+	assert(is_anchor_witness_script(script, tal_bytelen(script)));
 	return script;
 }
 
-bool is_to_remote_anchored_witness_script(const u8 *script, size_t script_len)
+bool is_anchor_witness_script(const u8 *script, size_t script_len)
 {
 	size_t len = 34 + 1 + 1 + 1;
 	/* With option_will_fund, the pushbytes can be up to 2 bytes more
@@ -476,8 +413,10 @@ u8 *p2wpkh_scriptcode(const tal_t *ctx, const struct pubkey *key)
 	return script;
 }
 
-bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
+bool is_p2pkh(const u8 *script, struct bitcoin_address *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2PKH_LEN)
 		return false;
 	if (script[0] != OP_DUP)
@@ -495,8 +434,10 @@ bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
 	return true;
 }
 
-bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr)
+bool is_p2sh(const u8 *script, struct ripemd160 *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2SH_LEN)
 		return false;
 	if (script[0] != OP_HASH160)
@@ -510,8 +451,10 @@ bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr)
 	return true;
 }
 
-bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr)
+bool is_p2wsh(const u8 *script, struct sha256 *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2WSH_LEN)
 		return false;
 	if (script[0] != OP_0)
@@ -523,8 +466,10 @@ bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr)
 	return true;
 }
 
-bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
+bool is_p2wpkh(const u8 *script, struct bitcoin_address *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN)
 		return false;
 	if (script[0] != OP_0)
@@ -536,34 +481,10 @@ bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr
 	return true;
 }
 
-bool is_p2tr(const u8 *script, size_t script_len, u8 xonly_pubkey[32])
+bool is_known_scripttype(const u8 *script)
 {
-	if (script_len != BITCOIN_SCRIPTPUBKEY_P2TR_LEN)
-		return false;
-	if (script[0] != OP_1)
-		return false;
-	/* x-only pubkey */
-	if (script[1] != OP_PUSHBYTES(32))
-		return false;
-	if (xonly_pubkey)
-		memcpy(xonly_pubkey, script+2, 32);
-	return true;
-}
-
-bool is_known_scripttype(const u8 *script, size_t script_len)
-{
-	return is_p2wpkh(script, script_len, NULL)
-		|| is_p2wsh(script, script_len, NULL)
-		|| is_p2sh(script, script_len, NULL)
-		|| is_p2pkh(script, script_len, NULL)
-		|| is_p2tr(script, script_len, NULL);
-}
-
-bool is_known_segwit_scripttype(const u8 *script, size_t script_len)
-{
-	return is_p2wpkh(script, script_len, NULL)
-		|| is_p2wsh(script, script_len, NULL)
-		|| is_p2tr(script, script_len, NULL);
+	return is_p2wpkh(script, NULL) || is_p2wsh(script, NULL)
+		|| is_p2sh(script, NULL) || is_p2pkh(script, NULL);
 }
 
 u8 **bitcoin_witness_sig_and_element(const tal_t *ctx,
@@ -643,7 +564,7 @@ u8 *bitcoin_wscript_to_local(const tal_t *ctx, u16 to_self_delay,
  * This output sends funds to either an HTLC-timeout transaction after the
  * HTLC-timeout or to the remote node using the payment preimage or the
  * revocation key. The output is a P2WSH, with a witness script (no
- * option_anchors):
+ * option_anchor_outputs):
  *
  *     # To remote node with revocation key
  *     OP_DUP OP_HASH160 <RIPEMD160(SHA256(revocationpubkey))> OP_EQUAL
@@ -661,7 +582,7 @@ u8 *bitcoin_wscript_to_local(const tal_t *ctx, u16 to_self_delay,
  *         OP_ENDIF
  *     OP_ENDIF
  *
- * Or, with `option_anchors`:
+ * Or, with `option_anchor_outputs`:
  *
  *  # To remote node with revocation key
  *  OP_DUP OP_HASH160 <RIPEMD160(SHA256(revocationpubkey))> OP_EQUAL
@@ -685,8 +606,7 @@ u8 *bitcoin_wscript_htlc_offer_ripemd160(const tal_t *ctx,
 					 const struct pubkey *remotehtlckey,
 					 const struct ripemd160 *payment_ripemd,
 					 const struct pubkey *revocationkey,
-					 bool option_anchor_outputs,
-					 bool option_anchors_zero_fee_htlc_tx)
+					 bool option_anchor_outputs)
 {
 	u8 *script = tal_arr(ctx, u8, 0);
 	struct ripemd160 ripemd;
@@ -718,7 +638,7 @@ u8 *bitcoin_wscript_htlc_offer_ripemd160(const tal_t *ctx,
 	add_op(&script, OP_EQUALVERIFY);
 	add_op(&script, OP_CHECKSIG);
 	add_op(&script, OP_ENDIF);
-	if (option_anchor_outputs || option_anchors_zero_fee_htlc_tx) {
+	if (option_anchor_outputs) {
 		add_number(&script, 1);
 		add_op(&script, OP_CHECKSEQUENCEVERIFY);
 		add_op(&script, OP_DROP);
@@ -733,8 +653,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
 			       const struct pubkey *remotehtlckey,
 			       const struct sha256 *payment_hash,
 			       const struct pubkey *revocationkey,
-			       bool option_anchor_outputs,
-			       bool option_anchors_zero_fee_htlc_tx)
+			       bool option_anchor_outputs)
 {
 	struct ripemd160 ripemd;
 
@@ -742,8 +661,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
 	return bitcoin_wscript_htlc_offer_ripemd160(ctx, localhtlckey,
 						    remotehtlckey,
 						    &ripemd, revocationkey,
-						    option_anchor_outputs,
-						    option_anchors_zero_fee_htlc_tx);
+						    option_anchor_outputs);
 }
 
 /* BOLT #3:
@@ -753,7 +671,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
  * This output sends funds to either the remote node after the HTLC-timeout or
  * using the revocation key, or to an HTLC-success transaction with a
  * successful payment preimage. The output is a P2WSH, with a witness script
- * (no `option_anchors`):
+ * (no `option_anchor_outputs`):
  *
  *     # To remote node with revocation key
  *     OP_DUP OP_HASH160 <RIPEMD160(SHA256(revocationpubkey))> OP_EQUAL
@@ -773,7 +691,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
  *         OP_ENDIF
  *     OP_ENDIF
  *
- * Or, with `option_anchors`:
+ * Or, with `option_anchor_outputs`:
  *
  *  # To remote node with revocation key
  *  OP_DUP OP_HASH160 <RIPEMD160(SHA256(revocationpubkey))> OP_EQUAL
@@ -799,8 +717,7 @@ u8 *bitcoin_wscript_htlc_receive_ripemd(const tal_t *ctx,
 					const struct pubkey *remotehtlckey,
 					const struct ripemd160 *payment_ripemd,
 					const struct pubkey *revocationkey,
-					bool option_anchor_outputs,
-					bool option_anchors_zero_fee_htlc_tx)
+					bool option_anchor_outputs)
 {
 	u8 *script = tal_arr(ctx, u8, 0);
 	struct ripemd160 ripemd;
@@ -835,7 +752,7 @@ u8 *bitcoin_wscript_htlc_receive_ripemd(const tal_t *ctx,
 	add_op(&script, OP_DROP);
 	add_op(&script, OP_CHECKSIG);
 	add_op(&script, OP_ENDIF);
-	if (option_anchor_outputs || option_anchors_zero_fee_htlc_tx) {
+	if (option_anchor_outputs) {
 		add_number(&script, 1);
 		add_op(&script, OP_CHECKSEQUENCEVERIFY);
 		add_op(&script, OP_DROP);
@@ -851,8 +768,7 @@ u8 *bitcoin_wscript_htlc_receive(const tal_t *ctx,
 				 const struct pubkey *remotehtlckey,
 				 const struct sha256 *payment_hash,
 				 const struct pubkey *revocationkey,
-				 bool option_anchor_outputs,
-				 bool option_anchors_zero_fee_htlc_tx)
+				 bool option_anchor_outputs)
 {
 	struct ripemd160 ripemd;
 
@@ -860,8 +776,7 @@ u8 *bitcoin_wscript_htlc_receive(const tal_t *ctx,
 	return bitcoin_wscript_htlc_receive_ripemd(ctx, htlc_abstimeout,
 						   localhtlckey, remotehtlckey,
 						   &ripemd, revocationkey,
-						   option_anchor_outputs,
-						   option_anchors_zero_fee_htlc_tx);
+						   option_anchor_outputs);
 }
 
 /* BOLT #3:
@@ -944,7 +859,7 @@ u8 *bitcoin_wscript_anchor(const tal_t *ctx,
 	u8 *script = tal_arr(ctx, u8, 0);
 
 	/* BOLT #3:
-	 * #### `to_local_anchor` and `to_remote_anchor` Output (option_anchors)
+	 * #### `to_local_anchor` and `to_remote_anchor` Output (option_anchor_outputs)
 	 *...
 	 *  <local_funding_pubkey/remote_funding_pubkey> OP_CHECKSIG OP_IFDUP
 	 *  OP_NOTIF
@@ -959,35 +874,15 @@ u8 *bitcoin_wscript_anchor(const tal_t *ctx,
 	add_op(&script, OP_CHECKSEQUENCEVERIFY);
 	add_op(&script, OP_ENDIF);
 
-	assert(is_anchor_witness_script(script, tal_bytelen(script)));
 	return script;
 }
 
-bool is_anchor_witness_script(const u8 *script, size_t script_len)
-{
-	if (script_len != 34 + 1 + 1 + 1 + 1 + 1 + 1)
-		return false;
-	if (script[0] != OP_PUSHBYTES(33))
-		return false;
-	if (script[34] != OP_CHECKSIG)
-		return false;
-	if (script[35] != OP_IFDUP)
-		return false;
-	if (script[36] != OP_NOTIF)
-		return false;
-	if (script[37] != 0x50 + 16)
-		return false;
-	if (script[38] != OP_CHECKSEQUENCEVERIFY)
-		return false;
-	if (script[39] != OP_ENDIF)
-		return false;
-	return true;
-}
-
 bool scripteq(const u8 *s1, const u8 *s2)
 {
-	size_t s1_len = tal_count(s1), s2_len = tal_count(s2);
-	memcheck(s1, s1_len);
-	memcheck(s2, s2_len);
-	return memeq(s1, s1_len, s2, s2_len);
+	memcheck(s1, tal_count(s1));
+	memcheck(s2, tal_count(s2));
+
+	if (tal_count(s1) != tal_count(s2))
+		return false;
+	return memcmp(s1, s2, tal_count(s1)) == 0;
 }

bitcoin/script.h

@@ -63,23 +63,10 @@ u8 *scriptpubkey_p2wpkh_derkey(const tal_t *ctx, const u8 der[33]);
 u8 *scriptpubkey_witness_raw(const tal_t *ctx, u8 version,
 			     const u8 *wprog, size_t wprog_size);
 
-/* Create an output script for a "raw"(perhaps already tweaked) taproot output pubkey */
-u8 *scriptpubkey_raw_p2tr(const tal_t *ctx, const struct pubkey *output_pubkey);
-
-/* Same as above, but compressed key is DER-encoded. */
-u8 *scriptpubkey_raw_p2tr_derkey(const tal_t *ctx, const u8 output_der[33]);
-
-/* Create an output script for an internal taproot pubkey. Results in different script than
- * scriptpubkey_raw_p2tr! TODO support merkle root tweaking */
-u8 *scriptpubkey_p2tr(const tal_t *ctx, const struct pubkey *inner_pubkey);
-
-/* Same as above, but compressed key is DER-encoded. TODO support merkle root tweaking */
-u8 *scriptpubkey_p2tr_derkey(const tal_t *ctx, const u8 inner_der[33]);
-
 /* To-remotekey with csv max(lease_expiry - blockheight, 1) delay. */
-u8 *bitcoin_wscript_to_remote_anchored(const tal_t *ctx,
-				       const struct pubkey *remote_key,
-				       u32 csv_lock);
+u8 *anchor_to_remote_redeem(const tal_t *ctx,
+			    const struct pubkey *remote_key,
+			    u32 csv_lock);
 
 /* Create a witness which spends the 2of2. */
 u8 **bitcoin_witness_2of2(const tal_t *ctx,
@@ -111,8 +98,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
 			       const struct pubkey *remotehtlckey,
 			       const struct sha256 *payment_hash,
 			       const struct pubkey *revocationkey,
-			       bool option_anchor_outputs,
-			       bool option_anchors_zero_fee_htlc_tx);
+			       bool option_anchor_outputs);
 u8 **bitcoin_witness_htlc_timeout_tx(const tal_t *ctx,
 				     const struct bitcoin_signature *localsig,
 				     const struct bitcoin_signature *remotesig,
@@ -123,8 +109,7 @@ u8 *bitcoin_wscript_htlc_receive(const tal_t *ctx,
 				 const struct pubkey *remotekey,
 				 const struct sha256 *payment_hash,
 				 const struct pubkey *revocationkey,
-				 bool option_anchor_outputs,
-				 bool option_anchors_zero_fee_htlc_tx);
+				 bool option_anchor_outputs);
 u8 **bitcoin_witness_htlc_success_tx(const tal_t *ctx,
 				     const struct bitcoin_signature *localsig,
 				     const struct bitcoin_signature *remotesig,
@@ -137,16 +122,14 @@ u8 *bitcoin_wscript_htlc_offer_ripemd160(const tal_t *ctx,
 					 const struct pubkey *remotehtlckey,
 					 const struct ripemd160 *payment_ripemd,
 					 const struct pubkey *revocationkey,
-					 bool option_anchor_outputs,
-					 bool option_anchors_zero_fee_htlc_tx);
+					 bool option_anchor_outputs);
 u8 *bitcoin_wscript_htlc_receive_ripemd(const tal_t *ctx,
 					const struct abs_locktime *htlc_abstimeout,
 					const struct pubkey *localkey,
 					const struct pubkey *remotekey,
 					const struct ripemd160 *payment_ripemd,
 					const struct pubkey *revocationkey,
-					bool option_anchor_outputs,
-					bool option_anchors_zero_fee_htlc_tx);
+					bool option_anchor_outputs);
 
 /* BOLT #3 HTLC-success/HTLC-timeout output */
 u8 *bitcoin_wscript_htlc_tx(const tal_t *ctx,
@@ -159,28 +142,19 @@ u8 *bitcoin_wscript_anchor(const tal_t *ctx,
 			   const struct pubkey *funding_pubkey);
 
 /* Is this a pay to pubkey hash? (extract addr if not NULL) */
-bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr);
+bool is_p2pkh(const u8 *script, struct bitcoin_address *addr);
 
 /* Is this a pay to script hash? (extract addr if not NULL) */
-bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr);
+bool is_p2sh(const u8 *script, struct ripemd160 *addr);
 
 /* Is this (version 0) pay to witness script hash? (extract addr if not NULL) */
-bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr);
+bool is_p2wsh(const u8 *script, struct sha256 *addr);
 
 /* Is this (version 0) pay to witness pubkey hash? (extract addr if not NULL) */
-bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr);
+bool is_p2wpkh(const u8 *script, struct bitcoin_address *addr);
 
-/* Is this a taproot output? (extract xonly_pubkey bytes if not NULL) */
-bool is_p2tr(const u8 *script, size_t script_len, u8 xonly_pubkey[32]);
-
-/* Is this one of the above script types? */
-bool is_known_scripttype(const u8 *script, size_t script_len);
-
-/* Is this a witness script type? */
-bool is_known_segwit_scripttype(const u8 *script, size_t script_len);
-
-/* Is this a to-remote witness script (used for option_anchor_outputs)? */
-bool is_to_remote_anchored_witness_script(const u8 *script, size_t script_len);
+/* Is this one of the four above script types? */
+bool is_known_scripttype(const u8 *script);
 
 /* Is this an anchor witness script? */
 bool is_anchor_witness_script(const u8 *script, size_t script_len);
@@ -203,7 +177,4 @@ void script_push_bytes(u8 **scriptp, const void *mem, size_t len);
 /* OP_0 + PUSH(32-byte-hash) */
 #define BITCOIN_SCRIPTPUBKEY_P2WSH_LEN (1 + 1 + 32)
 
-/* OP_1 + PUSH(32-byte-key) */
-#define BITCOIN_SCRIPTPUBKEY_P2TR_LEN (1 + 1 + 32)
-
 #endif /* LIGHTNING_BITCOIN_SCRIPT_H */

bitcoin/shadouble.c

@@ -1,7 +1,7 @@
 #include "config.h"
 #include <bitcoin/shadouble.h>
 #include <ccan/mem/mem.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
 void sha256_double(struct sha256_double *shadouble, const void *p, size_t len)
@@ -15,11 +15,7 @@ void sha256_double_done(struct sha256_ctx *shactx, struct sha256_double *res)
 	sha256_done(shactx, &res->sha);
 	sha256(&res->sha, &res->sha, sizeof(res->sha));
 }
-
-char *fmt_sha256_double(const tal_t *ctx, const struct sha256_double *shad)
-{
-	return tal_hexstr(ctx, shad, sizeof(*shad));
-}
+REGISTER_TYPE_TO_HEXSTR(sha256_double);
 
 void towire_sha256_double(u8 **pptr, const struct sha256_double *sha256d)
 {

bitcoin/shadouble.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/crypto/sha256/sha256.h>
 #include <ccan/short_types/short_types.h>
-#include <ccan/tal/tal.h>
 
 /* To explicitly distinguish between single sha and bitcoin's standard double */
 struct sha256_double {
@@ -18,6 +17,4 @@ void sha256_double_done(struct sha256_ctx *sha256, struct sha256_double *res);
 void fromwire_sha256_double(const u8 **cursor, size_t *max,
 			    struct sha256_double *sha256d);
 void towire_sha256_double(u8 **pptr, const struct sha256_double *sha256d);
-
-char *fmt_sha256_double(const tal_t *ctx, const struct sha256_double *shad);
 #endif /* LIGHTNING_BITCOIN_SHADOUBLE_H */

bitcoin/short_channel_id.c

@@ -1,6 +1,7 @@
 #include "config.h"
 #include <bitcoin/short_channel_id.h>
 #include <ccan/tal/str/str.h>
+#include <common/type_to_string.h>
 #include <stdio.h>
 #include <wire/wire.h>
 
@@ -52,7 +53,7 @@ bool short_channel_id_from_str(const char *str, size_t strlen,
 		&& mk_short_channel_id(dst, blocknum, txnum, outnum);
 }
 
-char *fmt_short_channel_id(const tal_t *ctx, struct short_channel_id scid)
+char *short_channel_id_to_str(const tal_t *ctx, const struct short_channel_id *scid)
 {
 	return tal_fmt(ctx, "%dx%dx%d",
 		       short_channel_id_blocknum(scid),
@@ -77,25 +78,26 @@ bool short_channel_id_dir_from_str(const char *str, size_t strlen,
 	return true;
 }
 
-char *fmt_short_channel_id_dir(const tal_t *ctx,
-			       const struct short_channel_id_dir *scidd)
+static char *short_channel_id_dir_to_str(const tal_t *ctx,
+					 const struct short_channel_id_dir *scidd)
 {
-	char *str, *scidstr = fmt_short_channel_id(NULL, scidd->scid);
+	char *str, *scidstr = short_channel_id_to_str(NULL, &scidd->scid);
 	str = tal_fmt(ctx, "%s/%u", scidstr, scidd->dir);
 	tal_free(scidstr);
 	return str;
 }
 
+REGISTER_TYPE_TO_STRING(short_channel_id, short_channel_id_to_str);
+REGISTER_TYPE_TO_STRING(short_channel_id_dir, short_channel_id_dir_to_str);
+
 void towire_short_channel_id(u8 **pptr,
-			     struct short_channel_id short_channel_id)
+			     const struct short_channel_id *short_channel_id)
 {
-	towire_u64(pptr, short_channel_id.u64);
+	towire_u64(pptr, short_channel_id->u64);
 }
 
-struct short_channel_id fromwire_short_channel_id(const u8 **cursor, size_t *max)
+void fromwire_short_channel_id(const u8 **cursor, size_t *max,
+			       struct short_channel_id *short_channel_id)
 {
-	struct short_channel_id scid;
-
-	scid.u64 = fromwire_u64(cursor, max);
-	return scid;
+	short_channel_id->u64 = fromwire_u64(cursor, max);
 }

bitcoin/short_channel_id.h

@@ -3,6 +3,7 @@
 #include "config.h"
 #include <ccan/compiler/compiler.h>
 #include <ccan/short_types/short_types.h>
+#include <ccan/structeq/structeq.h>
 #include <ccan/tal/tal.h>
 #include <common/gossip_constants.h>
 
@@ -11,18 +12,8 @@
 struct short_channel_id {
 	u64 u64;
 };
-
-static inline bool short_channel_id_eq(struct short_channel_id a,
-				       struct short_channel_id b)
-{
-	return a.u64 == b.u64;
-}
-
-static inline size_t short_channel_id_hash(struct short_channel_id scid)
-{
-	/* scids cost money to generate, so simple hash works here */
-	return (scid.u64 >> 32) ^ (scid.u64 >> 16) ^ scid.u64;
-}
+/* Define short_channel_id_eq (no padding) */
+STRUCTEQ_DEF(short_channel_id, 0, u64);
 
 /* BOLT #7:
  *
@@ -41,38 +32,25 @@ struct short_channel_id_dir {
 	int dir;
 };
 
-static inline bool short_channel_id_dir_eq(const struct short_channel_id_dir *a,
-					   const struct short_channel_id_dir *b)
+static inline u32 short_channel_id_blocknum(const struct short_channel_id *scid)
 {
-	return short_channel_id_eq(a->scid, b->scid) && a->dir == b->dir;
+	return scid->u64 >> 40;
 }
 
-static inline u32 short_channel_id_blocknum(struct short_channel_id scid)
+static inline u32 short_channel_id_txnum(const struct short_channel_id *scid)
 {
-	return scid.u64 >> 40;
+	return (scid->u64 >> 16) & 0x00FFFFFF;
 }
 
-static inline bool is_stub_scid(struct short_channel_id scid)
+static inline u16 short_channel_id_outnum(const struct short_channel_id *scid)
 {
-	return scid.u64 >> 40 == 1 &&
-		((scid.u64 >> 16) & 0x00FFFFFF) == 1 &&
-		(scid.u64 & 0xFFFF) == 1;
-}
-
-static inline u32 short_channel_id_txnum(struct short_channel_id scid)
-{
-	return (scid.u64 >> 16) & 0x00FFFFFF;
-}
-
-static inline u16 short_channel_id_outnum(struct short_channel_id scid)
-{
-	return scid.u64 & 0xFFFF;
+	return scid->u64 & 0xFFFF;
 }
 
 /* Subtly, at block N, depth is 1, hence the -1 here. eg. 103x1x0 is announceable
  * when height is 108. */
 static inline bool
-is_scid_depth_announceable(struct short_channel_id scid,
+is_scid_depth_announceable(const struct short_channel_id *scid,
 			  unsigned int height)
 {
 	return short_channel_id_blocknum(scid) + ANNOUNCE_MIN_DEPTH - 1
@@ -86,16 +64,15 @@ bool WARN_UNUSED_RESULT mk_short_channel_id(struct short_channel_id *scid,
 bool WARN_UNUSED_RESULT short_channel_id_from_str(const char *str, size_t strlen,
 						  struct short_channel_id *dst);
 
+char *short_channel_id_to_str(const tal_t *ctx, const struct short_channel_id *scid);
+
 bool WARN_UNUSED_RESULT short_channel_id_dir_from_str(const char *str, size_t strlen,
 						      struct short_channel_id_dir *scidd);
 
-char *fmt_short_channel_id(const tal_t *ctx, struct short_channel_id scid);
-char *fmt_short_channel_id_dir(const tal_t *ctx,
-			       const struct short_channel_id_dir *scidd);
-
 /* Marshal/unmarshal */
 void towire_short_channel_id(u8 **pptr,
-			     struct short_channel_id short_channel_id);
-struct short_channel_id fromwire_short_channel_id(const u8 **cursor, size_t *max);
+			     const struct short_channel_id *short_channel_id);
+void fromwire_short_channel_id(const u8 **cursor, size_t *max,
+			       struct short_channel_id *short_channel_id);
 
 #endif /* LIGHTNING_BITCOIN_SHORT_CHANNEL_ID_H */

bitcoin/signature.c

@@ -8,8 +8,7 @@
 #include <bitcoin/signature.h>
 #include <bitcoin/tx.h>
 #include <ccan/mem/mem.h>
-#include <common/utils.h>
-#include <secp256k1_schnorrsig.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
 #undef DEBUG
@@ -94,7 +93,12 @@ static bool sig_has_low_r(const secp256k1_ecdsa_signature* sig)
 	return compact_sig[0] < 0x80;
 }
 
-bool dev_no_signature_grind = false;
+#if DEVELOPER
+/* Some of the spec test vectors assume no sig grinding. */
+extern bool dev_no_grind;
+
+bool dev_no_grind = false;
+#endif
 
 void sign_hash(const struct privkey *privkey,
 	       const struct sha256_double *h,
@@ -109,10 +113,11 @@ void sign_hash(const struct privkey *privkey,
 					  s,
 					  h->sha.u.u8,
 					  privkey->secret.data, NULL,
-					  dev_no_signature_grind ? NULL
-					  : extra_entropy);
+					  IFDEV(dev_no_grind ? NULL
+						: extra_entropy,
+						extra_entropy));
 		((u32 *)extra_entropy)[0]++;
-		if (dev_no_signature_grind)
+		if (IFDEV(dev_no_grind, false))
 			break;
 	} while (!sig_has_low_r(s));
 
@@ -324,7 +329,7 @@ bool signature_from_der(const u8 *der, size_t len, struct bitcoin_signature *sig
 	return true;
 }
 
-char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx, const secp256k1_ecdsa_signature *sig)
+char *fmt_signature(const tal_t *ctx, const secp256k1_ecdsa_signature *sig)
 {
 	u8 der[72];
 	size_t len = 72;
@@ -334,15 +339,17 @@ char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx, const secp256k1_ecdsa_sign
 
 	return tal_hexstr(ctx, der, len);
 }
+REGISTER_TYPE_TO_STRING(secp256k1_ecdsa_signature, fmt_signature);
 
-char *fmt_bitcoin_signature(const tal_t *ctx,
-			    const struct bitcoin_signature *sig)
+static char *bitcoin_signature_to_hexstr(const tal_t *ctx,
+					 const struct bitcoin_signature *sig)
 {
 	u8 der[73];
 	size_t len = signature_to_der(der, sig);
 
 	return tal_hexstr(ctx, der, len);
 }
+REGISTER_TYPE_TO_STRING(bitcoin_signature, bitcoin_signature_to_hexstr);
 
 void fromwire_bitcoin_signature(const u8 **cursor, size_t *max,
 				struct bitcoin_signature *sig)
@@ -376,6 +383,8 @@ char *fmt_bip340sig(const tal_t *ctx, const struct bip340sig *bip340sig)
 	return tal_hexstr(ctx, bip340sig->u8, sizeof(bip340sig->u8));
 }
 
+REGISTER_TYPE_TO_HEXSTR(bip340sig);
+
 /* BIP-340:
  *
  * This proposal suggests to include the tag by prefixing the hashed
@@ -406,27 +415,3 @@ void bip340_sighash_init(struct sha256_ctx *sctx,
 	sha256_update(sctx, &taghash, sizeof(taghash));
 }
 
-
-bool check_schnorr_sig(const struct sha256 *hash,
-		       const secp256k1_pubkey *pubkey,
-		       const struct bip340sig *sig)
-{
-	/* FIXME: uuuugly!  There's no non-xonly verify function. */
-	u8 raw[PUBKEY_CMPR_LEN];
-	size_t outlen = sizeof(raw);
-	secp256k1_xonly_pubkey xonly_pubkey;
-
-	if (!secp256k1_ec_pubkey_serialize(secp256k1_ctx, raw, &outlen,
-					   pubkey,
-					   SECP256K1_EC_COMPRESSED))
-		abort();
-	assert(outlen == PUBKEY_CMPR_LEN);
-	if (!secp256k1_xonly_pubkey_parse(secp256k1_ctx, &xonly_pubkey, raw+1))
-		abort();
-
- 	return secp256k1_schnorrsig_verify(secp256k1_ctx,
- 					   sig->u8,
- 					   hash->u.u8,
- 					   sizeof(hash->u.u8),
-					   &xonly_pubkey) == 1;
-}

bitcoin/signature.h

@@ -5,14 +5,12 @@
 #include <ccan/tal/tal.h>
 #include <secp256k1.h>
 
-struct sha256;
 struct sha256_double;
 struct sha256_ctx;
 struct bitcoin_tx;
 struct pubkey;
 struct privkey;
 struct bitcoin_tx_output;
-struct bip340sig;
 
 enum sighash_type {
     SIGHASH_ALL = 1,
@@ -123,13 +121,6 @@ bool check_tx_sig(const struct bitcoin_tx *tx, size_t input_num,
 		  const struct pubkey *key,
 		  const struct bitcoin_signature *sig);
 
-/**
- * check a Schnorr signature
- */
-bool check_schnorr_sig(const struct sha256 *hash,
-		       const secp256k1_pubkey *pubkey,
-		       const struct bip340sig *sig);
-
 /* Give DER encoding of signature: returns length used (<= 73). */
 size_t signature_to_der(u8 der[73], const struct bitcoin_signature *sig);
 
@@ -150,11 +141,8 @@ void fromwire_bip340sig(const u8 **cursor, size_t *max,
 			struct bip340sig *bip340sig);
 
 /* Get a hex string sig */
-char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx,
-				    const secp256k1_ecdsa_signature *sig);
+char *fmt_signature(const tal_t *ctx, const secp256k1_ecdsa_signature *sig);
 char *fmt_bip340sig(const tal_t *ctx, const struct bip340sig *bip340sig);
-char *fmt_bitcoin_signature(const tal_t *ctx,
-			    const struct bitcoin_signature *sig);
 
 /* For caller convenience, we hand in tag in parts (any can be "") */
 void bip340_sighash_init(struct sha256_ctx *sctx,
@@ -162,7 +150,4 @@ void bip340_sighash_init(struct sha256_ctx *sctx,
 			 const char *tag2,
 			 const char *tag3);
 
-/* Some of the spec test vectors assume no sig grinding. */
-extern bool dev_no_signature_grind;
-
 #endif /* LIGHTNING_BITCOIN_SIGNATURE_H */

bitcoin/test/Makefile

@@ -4,7 +4,7 @@ BITCOIN_TEST_PROGRAMS := $(BITCOIN_TEST_OBJS:.o=)
 
 BITCOIN_TEST_COMMON_OBJS := common/utils.o common/setup.o common/autodata.o
 
-$(BITCOIN_TEST_PROGRAMS): $(BITCOIN_TEST_COMMON_OBJS) bitcoin/chainparams.o
+$(BITCOIN_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_TEST_COMMON_OBJS) bitcoin/chainparams.o
 $(BITCOIN_TEST_OBJS): $(CCAN_HEADERS) $(BITCOIN_HEADERS) $(BITCOIN_SRC)
 
 ALL_TEST_PROGRAMS += $(BITCOIN_TEST_PROGRAMS)

bitcoin/test/run-bitcoin_block_from_hex.c

@@ -2,7 +2,6 @@
 #include "../block.c"
 #include "../psbt.c"
 #include "../shadouble.c"
-#include "../signature.c"
 #include "../tx.c"
 #include "../varint.c"
 #include <assert.h>
@@ -16,9 +15,6 @@ bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
 /* Generated stub for amount_sat */
 struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 { fprintf(stderr, "amount_sat called!\n"); abort(); }
@@ -50,50 +46,33 @@ const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy
 /* Generated stub for fromwire_fail */
 void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_fail called!\n"); abort(); }
-/* Generated stub for fromwire_secp256k1_ecdsa_signature */
-void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-					secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); }
 /* Generated stub for fromwire_sha256 */
 void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED)
 { fprintf(stderr, "fromwire_sha256 called!\n"); abort(); }
 /* Generated stub for fromwire_u32 */
 u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_u32 called!\n"); abort(); }
-/* Generated stub for fromwire_u8 */
-u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); }
-/* Generated stub for fromwire_u8_array */
-void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED)
-{ fprintf(stderr, "fromwire_u8_array called!\n"); abort(); }
 /* Generated stub for is_anchor_witness_script */
 bool is_anchor_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
 { fprintf(stderr, "is_anchor_witness_script called!\n"); abort(); }
-/* Generated stub for is_to_remote_anchored_witness_script */
-bool is_to_remote_anchored_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_to_remote_anchored_witness_script called!\n"); abort(); }
 /* Generated stub for pubkey_to_der */
 void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UNNEEDED)
 { fprintf(stderr, "pubkey_to_der called!\n"); abort(); }
 /* Generated stub for pubkey_to_hash160 */
 void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
 { fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
+/* Generated stub for script_push_bytes */
+void script_push_bytes(u8 **scriptp UNNEEDED, const void *mem UNNEEDED, size_t len UNNEEDED)
+{ fprintf(stderr, "script_push_bytes called!\n"); abort(); }
 /* Generated stub for scriptpubkey_p2wsh */
 u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
 { fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }
-/* Generated stub for towire_secp256k1_ecdsa_signature */
-void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED,
-			      const secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); }
 /* Generated stub for towire_sha256 */
 void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED)
 { fprintf(stderr, "towire_sha256 called!\n"); abort(); }
 /* Generated stub for towire_u32 */
 void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED)
 { fprintf(stderr, "towire_u32 called!\n"); abort(); }
-/* Generated stub for towire_u8 */
-void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED)
-{ fprintf(stderr, "towire_u8 called!\n"); abort(); }
 /* Generated stub for towire_u8_array */
 void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED)
 { fprintf(stderr, "towire_u8_array called!\n"); abort(); }
@@ -171,13 +150,13 @@ int main(int argc, const char *argv[])
 				   block, strlen(block));
 
 	assert(b);
-	assert(b->hdr.version == 0x6592a000);
+	assert(b->hdr.version == CPU_TO_LE32(0x6592a000));
 	bitcoin_blkid_from_hex("0000000000000f31173e973bc00e452b1fac350066df7db2adec1e3224ea5bc1", strlen("0000000000000f31173e973bc00e452b1fac350066df7db2adec1e3224ea5bc1"), &prev);
 	assert(bitcoin_blkid_eq(&prev, &b->hdr.prev_hash));
 	hex_decode("8a0ee58ded5de949325ebc99583e3ca84f96a6597465c611685413f50f0ead7e", strlen("8a0ee58ded5de949325ebc99583e3ca84f96a6597465c611685413f50f0ead7e"), &merkle, sizeof(merkle));
 	assert(sha256_double_eq(&merkle, &b->hdr.merkle_hash));
-	assert(b->hdr.timestamp == 1550507183);
-	assert(b->hdr.nonce == 1226407989);
+	assert(b->hdr.timestamp == CPU_TO_LE32(1550507183));
+	assert(b->hdr.nonce == CPU_TO_LE32(1226407989));
 
 	assert(tal_count(b->tx) == 3);
 	bitcoin_txid(b->tx[0], &txid);

bitcoin/test/run-psbt-from-tx.c

@@ -1,121 +0,0 @@
-#include "config.h"
-#include "../psbt.c"
-#include "../tx.c"
-#include "../../wire/towire.c"
-#include "../../wire/fromwire.c"
-#include <assert.h>
-#include <ccan/cast/cast.h>
-#include <common/setup.h>
-#include <stdio.h>
-
-/* AUTOGENERATED MOCKS START */
-/* Generated stub for amount_asset_is_main */
-bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
-{ fprintf(stderr, "amount_asset_is_main called!\n"); abort(); }
-/* Generated stub for amount_asset_to_sat */
-struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
-{ fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
-/* Generated stub for amount_sat */
-struct amount_sat amount_sat(u64 satoshis UNNEEDED)
-{ fprintf(stderr, "amount_sat called!\n"); abort(); }
-/* Generated stub for amount_sat_add */
- bool amount_sat_add(struct amount_sat *val UNNEEDED,
-				       struct amount_sat a UNNEEDED,
-				       struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_add called!\n"); abort(); }
-/* Generated stub for amount_sat_eq */
-bool amount_sat_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_eq called!\n"); abort(); }
-/* Generated stub for amount_sat_greater_eq */
-bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); }
-/* Generated stub for amount_sat_sub */
- bool amount_sat_sub(struct amount_sat *val UNNEEDED,
-				       struct amount_sat a UNNEEDED,
-				       struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_sub called!\n"); abort(); }
-/* Generated stub for amount_sat_to_asset */
-struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u8 *asset UNNEEDED)
-{ fprintf(stderr, "amount_sat_to_asset called!\n"); abort(); }
-/* Generated stub for amount_tx_fee */
-struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_tx_fee called!\n"); abort(); }
-/* Generated stub for fromwire_sha256_double */
-void fromwire_sha256_double(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-			    struct sha256_double *sha256d UNNEEDED)
-{ fprintf(stderr, "fromwire_sha256_double called!\n"); abort(); }
-/* Generated stub for is_anchor_witness_script */
-bool is_anchor_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_anchor_witness_script called!\n"); abort(); }
-/* Generated stub for is_to_remote_anchored_witness_script */
-bool is_to_remote_anchored_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_to_remote_anchored_witness_script called!\n"); abort(); }
-/* Generated stub for pubkey_to_der */
-void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UNNEEDED)
-{ fprintf(stderr, "pubkey_to_der called!\n"); abort(); }
-/* Generated stub for pubkey_to_hash160 */
-void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
-{ fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
-/* Generated stub for scriptpubkey_p2wsh */
-u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
-{ fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }
-/* Generated stub for sha256_double */
-void sha256_double(struct sha256_double *shadouble UNNEEDED, const void *p UNNEEDED, size_t len UNNEEDED)
-{ fprintf(stderr, "sha256_double called!\n"); abort(); }
-/* Generated stub for signature_from_der */
-bool signature_from_der(const u8 *der UNNEEDED, size_t len UNNEEDED, struct bitcoin_signature *sig UNNEEDED)
-{ fprintf(stderr, "signature_from_der called!\n"); abort(); }
-/* Generated stub for signature_to_der */
-size_t signature_to_der(u8 der[73] UNNEEDED, const struct bitcoin_signature *sig UNNEEDED)
-{ fprintf(stderr, "signature_to_der called!\n"); abort(); }
-/* Generated stub for towire_sha256_double */
-void towire_sha256_double(u8 **pptr UNNEEDED, const struct sha256_double *sha256d UNNEEDED)
-{ fprintf(stderr, "towire_sha256_double called!\n"); abort(); }
-/* Generated stub for varint_put */
-size_t varint_put(u8 buf[VARINT_MAX_LEN] UNNEEDED, varint_t v UNNEEDED)
-{ fprintf(stderr, "varint_put called!\n"); abort(); }
-/* Generated stub for varint_size */
-size_t varint_size(varint_t v UNNEEDED)
-{ fprintf(stderr, "varint_size called!\n"); abort(); }
-/* AUTOGENERATED MOCKS END */
-
-/* This transaction has scriptSig data in it.
- * We expect that creating a new psbt from it will correctly
- * populate the PSBT object */
-static const char *raw_tx = "0200000000010151d12aa54cc6e59a6a92325a8315e93361d9805115a13aa5ba8dbcf30ffd858c000000001716001401fad90abcd66697e2592164722de4a95ebee165fdffffff02603c250200000000160014c2ccab171c2a5be9dab52ec41b825863024c546600093d00000000002200205b8cd3b914cf67cdd8fa6273c930353dd36476734fbd962102c2df53b90880cd02473044022001e73b1745d775521c758e70549ad79b1d076efc34303f416e66ff630f6088e402207b0aa44b35329ae4733463bc9f6ca433c5595f00a902a21c941945a24f8aa577012103d745445c9362665f22e0d96e9e766f273f3260dea39c8a76bfa05dd2684ddccf66000000";
-
-int main(int argc, char *argv[])
-{
-	struct bitcoin_tx *tx, *tx2;
-	u8 *msg;
-	size_t len;
-
-	common_setup(argv[0]);
-	chainparams = chainparams_for_network("bitcoin");
-
-	msg = tal_arr(tmpctx, u8, 0);
-	tx = bitcoin_tx_from_hex(tmpctx, raw_tx, strlen(raw_tx));
-
-	/* convert to wire format */
-	towire_bitcoin_tx(&msg, tx);
-
-	len = tal_bytelen(msg);
-	assert(len > 0);
-
-	tx2 = fromwire_bitcoin_tx(tmpctx,
-				  cast_const2(const u8 **, &msg), &len);
-	assert(tx2 != NULL);
-
-	/* Witness/scriptsig data is saved down into psbt */
-	assert(tx2->psbt->num_inputs == 1);
-	const struct wally_map_item *final_scriptsig = wally_map_get_integer(&tx2->psbt->inputs[0].psbt_fields, /* PSBT_IN_FINAL_SCRIPTSIG */ 0x07);
-	assert(final_scriptsig->value_len > 0);
-	assert(tx2->psbt->inputs[0].final_witness != NULL);
-	audit_psbt(tx2->psbt, tx2->psbt);
-
-	common_shutdown();
-	return 0;
-}

bitcoin/test/run-secret_eq_consttime.c

@@ -112,6 +112,7 @@ int main(int argc, char *argv[])
 {
 	const char *v;
 	int const_success, nonconst_success = ITERATIONS, i;
+	double load;
 
 	common_setup(argv[0]);
 
@@ -120,11 +121,6 @@ int main(int argc, char *argv[])
 	if (v && atoi(v) == 1)
 		goto exit;
 
-	/* this sometimes trips under CI */
-	v = getenv("SLOW_MACHINE");
-	if (v && atoi(v) == 1)
-		goto exit;
-
 	s1 = calloc(RUNS, sizeof(*s1));
 	s2 = calloc(RUNS, sizeof(*s2));
 
@@ -148,12 +144,17 @@ int main(int argc, char *argv[])
 	}
 
 	/* Now, check loadavg: if we weren't alone, that could explain results */
-	if (const_success < ITERATIONS / 2)
-		errx(1, "Only const time %u/%u?", const_success, i);
+	getloadavg(&load, 1);
+	if (load > 1.0) {
+		warnx("Load %.2f: too high, ignoring", load);
+	} else {
+		if (const_success < ITERATIONS / 2)
+			errx(1, "Only const time %u/%u?", const_success, i);
 
-	if (nonconst_success < ITERATIONS / 2)
-		errx(1, "memcmp seemed const time %u/%u?",
-		     nonconst_success, i);
+		if (nonconst_success < ITERATIONS / 2)
+			errx(1, "memcmp seemed const time %u/%u?",
+			     nonconst_success, i);
+	}
 	free(s1);
 	free(s2);
 

bitcoin/test/run-tx-bitcoin_tx_2of2_input_witness_weight.c

@@ -1,164 +0,0 @@
-#include "config.h"
-#include <assert.h>
-#include <bitcoin/privkey.c>
-#include <bitcoin/pubkey.c>
-#include <bitcoin/script.c>
-#include <bitcoin/shadouble.c>
-#include <bitcoin/signature.c>
-#include <bitcoin/tx.c>
-#include <bitcoin/varint.c>
-#include <ccan/str/hex/hex.h>
-#include <common/setup.h>
-#include <common/utils.h>
-#include <stdio.h>
-
-/* AUTOGENERATED MOCKS START */
-/* Generated stub for amount_asset_is_main */
-bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
-{ fprintf(stderr, "amount_asset_is_main called!\n"); abort(); }
-/* Generated stub for amount_asset_to_sat */
-struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
-{ fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
-/* Generated stub for amount_sat_add */
- bool amount_sat_add(struct amount_sat *val UNNEEDED,
-				       struct amount_sat a UNNEEDED,
-				       struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_add called!\n"); abort(); }
-/* Generated stub for amount_sat_greater_eq */
-bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); }
-/* Generated stub for amount_sat_sub */
- bool amount_sat_sub(struct amount_sat *val UNNEEDED,
-				       struct amount_sat a UNNEEDED,
-				       struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_sub called!\n"); abort(); }
-/* Generated stub for amount_sat_to_asset */
-struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u8 *asset UNNEEDED)
-{ fprintf(stderr, "amount_sat_to_asset called!\n"); abort(); }
-/* Generated stub for amount_tx_fee */
-struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_tx_fee called!\n"); abort(); }
-/* Generated stub for clone_psbt */
-struct wally_psbt *clone_psbt(const tal_t *ctx UNNEEDED, const struct wally_psbt *psbt UNNEEDED)
-{ fprintf(stderr, "clone_psbt called!\n"); abort(); }
-/* Generated stub for fromwire */
-const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED)
-{ fprintf(stderr, "fromwire called!\n"); abort(); }
-/* Generated stub for fromwire_fail */
-void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_fail called!\n"); abort(); }
-/* Generated stub for fromwire_secp256k1_ecdsa_signature */
-void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-					secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); }
-/* Generated stub for fromwire_sha256 */
-void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED)
-{ fprintf(stderr, "fromwire_sha256 called!\n"); abort(); }
-/* Generated stub for fromwire_u32 */
-u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_u32 called!\n"); abort(); }
-/* Generated stub for fromwire_u8 */
-u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); }
-/* Generated stub for fromwire_u8_array */
-void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED)
-{ fprintf(stderr, "fromwire_u8_array called!\n"); abort(); }
-/* Generated stub for fromwire_wally_psbt */
-struct wally_psbt *fromwire_wally_psbt(const tal_t *ctx UNNEEDED,
-				       const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_wally_psbt called!\n"); abort(); }
-/* Generated stub for new_psbt */
-struct wally_psbt *new_psbt(const tal_t *ctx UNNEEDED,
-			    const struct wally_tx *wtx UNNEEDED)
-{ fprintf(stderr, "new_psbt called!\n"); abort(); }
-/* Generated stub for psbt_add_output */
-struct wally_psbt_output *psbt_add_output(struct wally_psbt *psbt UNNEEDED,
-					  struct wally_tx_output *output UNNEEDED,
-					  size_t insert_at UNNEEDED)
-{ fprintf(stderr, "psbt_add_output called!\n"); abort(); }
-/* Generated stub for psbt_append_input */
-struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt UNNEEDED,
-					   const struct bitcoin_outpoint *outpoint UNNEEDED,
-					   u32 sequence UNNEEDED,
-					   const u8 *scriptSig UNNEEDED,
-					   const u8 *input_wscript UNNEEDED,
-					   const u8 *redeemscript UNNEEDED)
-{ fprintf(stderr, "psbt_append_input called!\n"); abort(); }
-/* Generated stub for psbt_final_tx */
-struct wally_tx *psbt_final_tx(const tal_t *ctx UNNEEDED, const struct wally_psbt *psbt UNNEEDED)
-{ fprintf(stderr, "psbt_final_tx called!\n"); abort(); }
-/* Generated stub for psbt_finalize */
-bool psbt_finalize(struct wally_psbt *psbt UNNEEDED)
-{ fprintf(stderr, "psbt_finalize called!\n"); abort(); }
-/* Generated stub for psbt_input_get_amount */
-struct amount_sat psbt_input_get_amount(const struct wally_psbt *psbt UNNEEDED,
-					size_t in UNNEEDED)
-{ fprintf(stderr, "psbt_input_get_amount called!\n"); abort(); }
-/* Generated stub for psbt_input_set_wit_utxo */
-void psbt_input_set_wit_utxo(struct wally_psbt *psbt UNNEEDED, size_t in UNNEEDED,
-			     const u8 *scriptPubkey UNNEEDED, struct amount_sat amt UNNEEDED)
-{ fprintf(stderr, "psbt_input_set_wit_utxo called!\n"); abort(); }
-/* Generated stub for towire */
-void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED)
-{ fprintf(stderr, "towire called!\n"); abort(); }
-/* Generated stub for towire_secp256k1_ecdsa_signature */
-void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED,
-			      const secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); }
-/* Generated stub for towire_sha256 */
-void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED)
-{ fprintf(stderr, "towire_sha256 called!\n"); abort(); }
-/* Generated stub for towire_u32 */
-void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED)
-{ fprintf(stderr, "towire_u32 called!\n"); abort(); }
-/* Generated stub for towire_u8 */
-void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED)
-{ fprintf(stderr, "towire_u8 called!\n"); abort(); }
-/* Generated stub for towire_u8_array */
-void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED)
-{ fprintf(stderr, "towire_u8_array called!\n"); abort(); }
-/* Generated stub for towire_wally_psbt */
-void towire_wally_psbt(u8 **pptr UNNEEDED, const struct wally_psbt *psbt UNNEEDED)
-{ fprintf(stderr, "towire_wally_psbt called!\n"); abort(); }
-/* AUTOGENERATED MOCKS END */
-
-int main(int argc, const char *argv[])
-{
-	struct privkey p;
-	struct pubkey pub;
-	struct bitcoin_signature sig;
-	struct sha256_double h;
-	u8 **wit;
-	size_t weight;
-
-	common_setup(argv[0]);
-	chainparams = chainparams_for_network("bitcoin");
-
-	memset(&h, 0, sizeof(h));
-	memset(&p, 1, sizeof(p));
-	sign_hash(&p, &h, &sig.s);
-	sig.sighash_type = SIGHASH_ALL;
-	pubkey_from_privkey(&p, &pub);
-
-	wit = bitcoin_witness_2of2(tmpctx, &sig, &sig, &pub, &pub);
-
-	/* 1 byte for num witnesses, one per witness element */
-	weight = 1;
-
-	/* Two signatures, slightly overestimated to be 73 bytes each,
-	 * while the actual witness will often be smaller.*/
-        /* BOLT #03:
-         * Signatures are 73 bytes long (the maximum length).
-         */
-	weight += 2 + 2;
-
-	for (size_t i = 0; i < tal_count(wit); i++)
-		weight += 1 + tal_bytelen(wit[i]);
-	assert(bitcoin_tx_2of2_input_witness_weight() == weight);
-
-	common_shutdown();
-	return 0;
-}

bitcoin/test/run-tx-encode.c

@@ -2,7 +2,6 @@
 #include <assert.h>
 #include <bitcoin/psbt.c>
 #include <bitcoin/shadouble.c>
-#include <bitcoin/signature.c>
 #include <bitcoin/tx.c>
 #include <bitcoin/varint.c>
 #include <ccan/str/hex/hex.h>
@@ -17,9 +16,6 @@ bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
 /* Generated stub for amount_sat */
 struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 { fprintf(stderr, "amount_sat called!\n"); abort(); }
@@ -51,50 +47,33 @@ const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy
 /* Generated stub for fromwire_fail */
 void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_fail called!\n"); abort(); }
-/* Generated stub for fromwire_secp256k1_ecdsa_signature */
-void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-					secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); }
 /* Generated stub for fromwire_sha256 */
 void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED)
 { fprintf(stderr, "fromwire_sha256 called!\n"); abort(); }
 /* Generated stub for fromwire_u32 */
 u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_u32 called!\n"); abort(); }
-/* Generated stub for fromwire_u8 */
-u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); }
-/* Generated stub for fromwire_u8_array */
-void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED)
-{ fprintf(stderr, "fromwire_u8_array called!\n"); abort(); }
 /* Generated stub for is_anchor_witness_script */
 bool is_anchor_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
 { fprintf(stderr, "is_anchor_witness_script called!\n"); abort(); }
-/* Generated stub for is_to_remote_anchored_witness_script */
-bool is_to_remote_anchored_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_to_remote_anchored_witness_script called!\n"); abort(); }
 /* Generated stub for pubkey_to_der */
 void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UNNEEDED)
 { fprintf(stderr, "pubkey_to_der called!\n"); abort(); }
 /* Generated stub for pubkey_to_hash160 */
 void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
 { fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
+/* Generated stub for script_push_bytes */
+void script_push_bytes(u8 **scriptp UNNEEDED, const void *mem UNNEEDED, size_t len UNNEEDED)
+{ fprintf(stderr, "script_push_bytes called!\n"); abort(); }
 /* Generated stub for scriptpubkey_p2wsh */
 u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
 { fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }
-/* Generated stub for towire_secp256k1_ecdsa_signature */
-void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED,
-			      const secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); }
 /* Generated stub for towire_sha256 */
 void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED)
 { fprintf(stderr, "towire_sha256 called!\n"); abort(); }
 /* Generated stub for towire_u32 */
 void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED)
 { fprintf(stderr, "towire_u32 called!\n"); abort(); }
-/* Generated stub for towire_u8 */
-void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED)
-{ fprintf(stderr, "towire_u8 called!\n"); abort(); }
 /* Generated stub for towire_u8_array */
 void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED)
 { fprintf(stderr, "towire_u8_array called!\n"); abort(); }

bitcoin/tx.c

@@ -1,14 +1,16 @@
 #include "config.h"
 #include <assert.h>
+#include <bitcoin/chainparams.h>
 #include <bitcoin/psbt.h>
 #include <bitcoin/script.h>
-#include <bitcoin/tx.h>
-#include <ccan/cast/cast.h>
 #include <ccan/str/hex/hex.h>
 #include <ccan/tal/str/str.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wally_psbt.h>
 #include <wire/wire.h>
+#include <bitcoin/tx.h>
+
+#define SEGREGATED_WITNESS_FLAG 0x1
 
 struct bitcoin_tx_output *new_tx_output(const tal_t *ctx,
 					struct amount_sat amount,
@@ -55,12 +57,12 @@ struct wally_tx_output *wally_tx_output(const tal_t *ctx,
 	}
 
 done:
-	tal_wally_end_onto(ctx, output, struct wally_tx_output);
+	tal_wally_end(tal_steal(ctx, output));
 	return output;
 }
 
 int bitcoin_tx_add_output(struct bitcoin_tx *tx, const u8 *script,
-			  const u8 *wscript, struct amount_sat amount)
+			  u8 *wscript, struct amount_sat amount)
 {
 	size_t i = tx->wtx->num_outputs;
 	struct wally_tx_output *output;
@@ -95,15 +97,6 @@ int bitcoin_tx_add_output(struct bitcoin_tx *tx, const u8 *script,
 	return i;
 }
 
-void bitcoin_tx_remove_output(struct bitcoin_tx *tx, size_t outnum)
-{
-	int ret;
-	ret = wally_tx_remove_output(tx->wtx, outnum);
-	assert(ret == WALLY_OK);
-	ret = wally_psbt_remove_output(tx->psbt, outnum);
-	assert(ret == WALLY_OK);
-}
-
 bool elements_wtx_output_is_fee(const struct wally_tx *tx, int outnum)
 {
 	assert(outnum < tx->num_outputs);
@@ -170,7 +163,7 @@ static int elements_tx_add_fee_output(struct bitcoin_tx *tx)
 	int pos;
 
 	/* If we aren't using elements, we don't add explicit fee outputs */
-	if (!chainparams->is_elements)
+	if (!chainparams->is_elements || amount_sat_eq(fee, AMOUNT_SAT(0)))
 		return -1;
 
 	/* Try to find any existing fee output */
@@ -190,36 +183,7 @@ static int elements_tx_add_fee_output(struct bitcoin_tx *tx)
 void bitcoin_tx_set_locktime(struct bitcoin_tx *tx, u32 locktime)
 {
 	tx->wtx->locktime = locktime;
-	tx->psbt->fallback_locktime = locktime;
-	tx->psbt->has_fallback_locktime = true;
-}
-
-/* FIXME Stolen from psbt_append_input; export? */
-static struct wally_tx_input *wally_tx_input_from_outpoint_sequence(const struct bitcoin_outpoint *outpoint,
-			u32 sequence)
-{
-	struct wally_tx_input *tx_in;
-    if (chainparams->is_elements) {
-        if (wally_tx_elements_input_init_alloc(outpoint->txid.shad.sha.u.u8,
-                               sizeof(outpoint->txid.shad.sha.u.u8),
-                               outpoint->n,
-                               sequence, NULL, 0,
-                               NULL,
-                               NULL, 0,
-                               NULL, 0, NULL, 0,
-                               NULL, 0, NULL, 0,
-                               NULL, 0, NULL,
-                               &tx_in) != WALLY_OK)
-            abort();
-    } else {
-        if (wally_tx_input_init_alloc(outpoint->txid.shad.sha.u.u8,
-                          sizeof(outpoint->txid.shad.sha.u.u8),
-                          outpoint->n,
-                          sequence, NULL, 0, NULL,
-                          &tx_in) != WALLY_OK)
-            abort();
-    }
-	return tx_in;
+	tx->psbt->tx->locktime = locktime;
 }
 
 int bitcoin_tx_add_input(struct bitcoin_tx *tx,
@@ -230,14 +194,13 @@ int bitcoin_tx_add_input(struct bitcoin_tx *tx,
 {
 	int wally_err;
 	int input_num = tx->wtx->num_inputs;
-	struct wally_tx_input *tx_input;
 
 	psbt_append_input(tx->psbt, outpoint,
 			  sequence, scriptSig,
 			  input_wscript, NULL);
 
 	if (input_wscript) {
-		scriptPubkey = scriptpubkey_p2wsh(tmpctx, input_wscript);
+		scriptPubkey = scriptpubkey_p2wsh(tx->psbt, input_wscript);
 	}
 
 	assert(scriptPubkey);
@@ -245,11 +208,9 @@ int bitcoin_tx_add_input(struct bitcoin_tx *tx,
 				scriptPubkey, amount);
 
 	tal_wally_start();
-	tx_input = wally_tx_input_from_outpoint_sequence(outpoint, sequence);
 	wally_err = wally_tx_add_input(tx->wtx,
-			       tx_input);
+				       &tx->psbt->tx->inputs[input_num]);
 	assert(wally_err == WALLY_OK);
-	wally_tx_input_free(tx_input);
 
 	/* scriptsig isn't actually stored in psbt input, so add that now */
 	wally_tx_set_input_script(tx->wtx, input_num,
@@ -257,10 +218,12 @@ int bitcoin_tx_add_input(struct bitcoin_tx *tx,
 	tal_wally_end(tx->wtx);
 
 	if (is_elements(chainparams)) {
+		struct amount_asset asset;
 		/* FIXME: persist asset tags */
-		amount_sat_to_asset(&amount,
+		asset = amount_sat_to_asset(&amount,
 					    chainparams->fee_asset_tag);
 		/* FIXME: persist nonces */
+		psbt_elements_input_set_asset(tx->psbt, input_num, &asset);
 	}
 	return input_num;
 }
@@ -298,24 +261,47 @@ void bitcoin_tx_output_set_amount(struct bitcoin_tx *tx, int outnum,
 		assert(ret == WALLY_OK);
 	} else {
 		output->satoshi = satoshis;
+
+		/* update the global tx for the psbt also */
+		output = &tx->psbt->tx->outputs[outnum];
+		output->satoshi = satoshis;
 	}
-	wally_psbt_output_set_amount(&tx->psbt->outputs[outnum], satoshis);
+}
+
+const u8 *wally_tx_output_get_script(const tal_t *ctx,
+				     const struct wally_tx_output *output)
+{
+	if (output->script == NULL) {
+		/* This can happen for coinbase transactions and pegin
+		 * transactions */
+		return NULL;
+	}
+
+	return tal_dup_arr(ctx, u8, output->script, output->script_len, 0);
+}
+
+const u8 *bitcoin_tx_output_get_script(const tal_t *ctx,
+				       const struct bitcoin_tx *tx, int outnum)
+{
+	const struct wally_tx_output *output;
+	assert(outnum < tx->wtx->num_outputs);
+	output = &tx->wtx->outputs[outnum];
+
+	return wally_tx_output_get_script(ctx, output);
 }
 
 u8 *bitcoin_tx_output_get_witscript(const tal_t *ctx, const struct bitcoin_tx *tx,
 				    int outnum)
 {
 	struct wally_psbt_output *out;
-	const struct wally_map_item *output_witness_script;
 
 	assert(outnum < tx->psbt->num_outputs);
 	out = &tx->psbt->outputs[outnum];
 
-	output_witness_script = wally_map_get_integer(&out->psbt_fields, /* PSBT_OUT_WITNESS_SCRIPT */ 0x01);
-	if (output_witness_script->value_len == 0)
+	if (out->witness_script_len == 0)
 		return NULL;
 
-	return tal_dup_arr(ctx, u8, output_witness_script->value, output_witness_script->value_len, 0);
+	return tal_dup_arr(ctx, u8, out->witness_script, out->witness_script_len, 0);
 }
 
 struct amount_asset bitcoin_tx_output_get_amount(const struct bitcoin_tx *tx,
@@ -403,7 +389,8 @@ void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
 	assert(innum < tx->wtx->num_inputs);
 
 	in = &tx->wtx->inputs[innum];
-	CROSS_TYPE_ASSIGNMENT(&in->txhash, &outpoint->txid);
+	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+	memcpy(in->txhash, &outpoint->txid, sizeof(struct bitcoin_txid));
 	in->index = outpoint->n;
 }
 
@@ -411,13 +398,15 @@ void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
 void wally_tx_input_get_txid(const struct wally_tx_input *in,
 			     struct bitcoin_txid *txid)
 {
-	CROSS_TYPE_ASSIGNMENT(txid, &in->txhash);
+	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+	memcpy(txid, in->txhash, sizeof(struct bitcoin_txid));
 }
 
 void wally_tx_input_get_outpoint(const struct wally_tx_input *in,
 				 struct bitcoin_outpoint *outpoint)
 {
-	wally_tx_input_get_txid(in, &outpoint->txid);
+	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+	memcpy(&outpoint->txid, in->txhash, sizeof(struct bitcoin_txid));
 	outpoint->n = in->index;
 }
 
@@ -468,21 +457,7 @@ size_t wally_tx_weight(const struct wally_tx *wtx)
 
 size_t bitcoin_tx_weight(const struct bitcoin_tx *tx)
 {
-	size_t extra;
-	size_t num_witnesses;
-
-	/* If we don't have witnesses *yet*, libwally doesn't encode
-	 * in BIP 141 style, omitting the flag and marker bytes */
-	wally_tx_get_witness_count(tx->wtx, &num_witnesses);
-	if (num_witnesses == 0) {
-		if (chainparams->is_elements)
-			extra = 7;
-		else
-			extra = 2;
-	} else
-		extra = 0;
-
-	return extra + wally_tx_weight(tx->wtx);
+	return wally_tx_weight(tx->wtx);
 }
 
 void wally_txid(const struct wally_tx *wtx, struct bitcoin_txid *txid)
@@ -534,7 +509,7 @@ struct bitcoin_tx *bitcoin_tx(const tal_t *ctx,
 	wally_tx_init_alloc(WALLY_TX_VERSION_2, nlocktime, input_count, output_count,
 			    &tx->wtx);
 	tal_add_destructor(tx, bitcoin_tx_destroy);
-	tal_wally_end_onto(tx, tx->wtx, struct wally_tx);
+	tal_wally_end(tal_steal(tx, tx->wtx));
 
 	tx->chainparams = chainparams;
 	tx->psbt = new_psbt(tx, tx->wtx);
@@ -548,28 +523,21 @@ void bitcoin_tx_finalize(struct bitcoin_tx *tx)
 	assert(bitcoin_tx_check(tx));
 }
 
-struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt TAKES)
+struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt STEALS)
 {
-	size_t locktime;
-
-	if (!taken(psbt))
-		psbt = clone_psbt(tmpctx, psbt);
-
-	wally_psbt_get_locktime(psbt, &locktime);
 	struct bitcoin_tx *tx = bitcoin_tx(ctx, chainparams,
-					   psbt->num_inputs,
-					   psbt->num_outputs,
-					   locktime);
+					   psbt->tx->num_inputs,
+					   psbt->tx->num_outputs,
+					   psbt->tx->locktime);
 	wally_tx_free(tx->wtx);
 
 	psbt_finalize(psbt);
 	tx->wtx = psbt_final_tx(tx, psbt);
 	if (!tx->wtx) {
 		tal_wally_start();
-		if (wally_psbt_extract(psbt, WALLY_PSBT_EXTRACT_NON_FINAL, &tx->wtx) != WALLY_OK) {
+		if (wally_tx_clone_alloc(psbt->tx, 0, &tx->wtx) != WALLY_OK)
 			tx->wtx = NULL;
-		}
-		tal_wally_end_onto(tx, tx->wtx, struct wally_tx);
+		tal_wally_end(tal_steal(tx, tx->wtx));
 		if (!tx->wtx)
 			return tal_free(tx);
 	}
@@ -580,30 +548,6 @@ struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psb
 	return tx;
 }
 
-struct bitcoin_tx *clone_bitcoin_tx(const tal_t *ctx,
-				    const struct bitcoin_tx *tx)
-{
-	struct bitcoin_tx *newtx;
-
-	if (taken(tx))
-		return cast_const(struct bitcoin_tx *, tal_steal(ctx, tx));
-
-	newtx = tal(ctx, struct bitcoin_tx);
-
-	newtx->chainparams = tx->chainparams;
-
-	tal_wally_start();
-	if (wally_tx_clone_alloc(tx->wtx, 0, &newtx->wtx) != WALLY_OK)
-		newtx->wtx = NULL;
-	tal_wally_end_onto(newtx, newtx->wtx, struct wally_tx);
-	if (!newtx->wtx)
-		return tal_free(newtx);
-
-	newtx->psbt = clone_psbt(newtx, tx->psbt);
-	tal_add_destructor(newtx, bitcoin_tx_destroy);
-	return newtx;
-}
-
 static struct wally_tx *pull_wtx(const tal_t *ctx,
 				 const u8 **cursor,
 				 size_t *max)
@@ -619,7 +563,7 @@ static struct wally_tx *pull_wtx(const tal_t *ctx,
 		fromwire_fail(cursor, max);
 		wtx = tal_free(wtx);
 	}
-	tal_wally_end_onto(ctx, wtx, struct wally_tx);
+	tal_wally_end(tal_steal(ctx, wtx));
 
 	if (wtx) {
 		size_t wsize;
@@ -631,8 +575,8 @@ static struct wally_tx *pull_wtx(const tal_t *ctx,
 	return wtx;
 }
 
-struct bitcoin_tx *pull_bitcoin_tx_only(const tal_t *ctx, const u8 **cursor,
-					size_t *max)
+struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx, const u8 **cursor,
+				   size_t *max)
 {
 	struct bitcoin_tx *tx = tal(ctx, struct bitcoin_tx);
 	tx->wtx = pull_wtx(tx, cursor, max);
@@ -641,23 +585,13 @@ struct bitcoin_tx *pull_bitcoin_tx_only(const tal_t *ctx, const u8 **cursor,
 
 	tal_add_destructor(tx, bitcoin_tx_destroy);
 	tx->chainparams = chainparams;
-	tx->psbt = NULL;
+	tx->psbt = new_psbt(tx, tx->wtx);
+	if (!tx->psbt)
+		return tal_free(tx);
 
 	return tx;
 }
 
-struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx, const u8 **cursor,
-				   size_t *max)
-{
-	struct bitcoin_tx *tx = pull_bitcoin_tx_only(ctx, cursor, max);
-	if (tx) {
-		tx->psbt = new_psbt(tx, tx->wtx);
-		if (!tx->psbt)
-			return tal_free(tx);
-        }
-	return tx;
-}
-
 struct bitcoin_tx *bitcoin_tx_from_hex(const tal_t *ctx, const char *hex,
 				       size_t hexlen)
 {
@@ -694,6 +628,18 @@ fail:
 	return NULL;
 }
 
+/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
+static void reverse_bytes(u8 *arr, size_t len)
+{
+	unsigned int i;
+
+	for (i = 0; i < len / 2; i++) {
+		unsigned char tmp = arr[i];
+		arr[i] = arr[len - 1 - i];
+		arr[len - 1 - i] = tmp;
+	}
+}
+
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid)
 {
@@ -711,7 +657,7 @@ bool bitcoin_txid_to_hex(const struct bitcoin_txid *txid,
 	return hex_encode(&rev, sizeof(rev), hexstr, hexstr_len);
 }
 
-char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx)
+static char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx)
 {
 	u8 *lin = linearize_tx(ctx, tx);
 	char *s = tal_hex(ctx, lin);
@@ -719,7 +665,7 @@ char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx)
 	return s;
 }
 
-char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
+static char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
 {
 	char *hexstr = tal_arr(ctx, char, hex_str_size(sizeof(*txid)));
 
@@ -727,15 +673,15 @@ char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
 	return hexstr;
 }
 
-char *fmt_bitcoin_outpoint(const tal_t *ctx,
-			   const struct bitcoin_outpoint *outpoint)
+static char *fmt_bitcoin_outpoint(const tal_t *ctx,
+				  const struct bitcoin_outpoint *outpoint)
 {
 	return tal_fmt(ctx, "%s:%u",
 		       fmt_bitcoin_txid(tmpctx, &outpoint->txid),
 		       outpoint->n);
 }
 
-char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
+static char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
 {
 	u8 *lin = linearize_wtx(ctx, tx);
 	char *s = tal_hex(ctx, lin);
@@ -743,15 +689,10 @@ char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
 	return s;
 }
 
-char *fmt_sha256(const tal_t *ctx, const struct sha256 *sha256)
-{
-	return tal_hexstr(ctx, sha256, sizeof(*sha256));
-}
-
-char *fmt_ripemd160(const tal_t *ctx, const struct ripemd160 *ripemd160)
-{
-	return tal_hexstr(ctx, ripemd160, sizeof(*ripemd160));
-}
+REGISTER_TYPE_TO_STRING(bitcoin_tx, fmt_bitcoin_tx);
+REGISTER_TYPE_TO_STRING(bitcoin_txid, fmt_bitcoin_txid);
+REGISTER_TYPE_TO_STRING(bitcoin_outpoint, fmt_bitcoin_outpoint);
+REGISTER_TYPE_TO_STRING(wally_tx, fmt_wally_tx);
 
 void fromwire_bitcoin_txid(const u8 **cursor, size_t *max,
 			   struct bitcoin_txid *txid)
@@ -766,13 +707,15 @@ struct bitcoin_tx *fromwire_bitcoin_tx(const tal_t *ctx,
 
 	u32 len = fromwire_u32(cursor, max);
 	size_t start = *max;
-	tx = pull_bitcoin_tx_only(ctx, cursor, max);
+	tx = pull_bitcoin_tx(ctx, cursor, max);
 	if (!tx)
 		return fromwire_fail(cursor, max);
 	// Check that we consumed len bytes
 	if (start - *max != len)
 		return fromwire_fail(cursor, max);
 
+	/* pull_bitcoin_tx sets the psbt */
+	tal_free(tx->psbt);
 	tx->psbt = fromwire_wally_psbt(tx, cursor, max);
 	if (!tx->psbt)
 		return fromwire_fail(cursor, max);
@@ -813,7 +756,8 @@ bool wally_tx_input_spends(const struct wally_tx_input *input,
 	/* Useful, as tx_part can have some NULL inputs */
 	if (!input)
 		return false;
-	if (!CROSS_TYPE_EQ(&outpoint->txid, &input->txhash))
+	BUILD_ASSERT(sizeof(outpoint->txid) == sizeof(input->txhash));
+	if (memcmp(&outpoint->txid, input->txhash, sizeof(outpoint->txid)) != 0)
 		return false;
 	return input->index == outpoint->n;
 }
@@ -897,8 +841,7 @@ size_t bitcoin_tx_input_sig_weight(void)
 /* Input weight */
 size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight)
 {
-	/* We assume < 253 witness elements */
-	size_t weight = 1 + witness_weight;
+	size_t weight = witness_weight;
 
 	/* Input weight: txid + index + sequence */
 	weight += (32 + 4 + 4) * 4;
@@ -917,74 +860,35 @@ size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight)
 	return weight;
 }
 
-size_t bitcoin_tx_input_witness_weight(enum utxotype utxotype)
+size_t bitcoin_tx_simple_input_witness_weight(void)
 {
-	switch (utxotype) {
-	case UTXO_P2SH_P2WPKH:
-	case UTXO_P2WPKH:
-		/* Account for witness (sig + key) */
-		return bitcoin_tx_input_sig_weight() + 1 + 33;
-	case UTXO_P2WSH_FROM_CLOSE:
-		/* BOLT #3:
-		 * #### `to_remote` Output
-		 *
-		 * If `option_anchors` applies to the commitment
-		 * transaction, the `to_remote` output is encumbered by a one
-		 * block csv lock.
-		 *    <remotepubkey> OP_CHECKSIGVERIFY 1 OP_CHECKSEQUENCEVERIFY
-		 *
-		 * The output is spent by an input with `nSequence` field set
-		 * to `1` and witness: <remote_sig>
-		 * Otherwise, this output is a simple P2WPKH to `remotepubkey`.
-		 */
-		/* In practice, these predate anchors, so: */
-		return 1 + 1 + bitcoin_tx_input_sig_weight();
-	case UTXO_P2TR:
-		return 1 + 64;
-	}
-	abort();
+	/* Account for witness (1 byte count + sig + key) */
+	return 1 + (bitcoin_tx_input_sig_weight() + 1 + 33);
 }
 
-size_t bitcoin_tx_2of2_input_witness_weight(void)
+/* We only do segwit inputs, and we assume witness is sig + key  */
+size_t bitcoin_tx_simple_input_weight(bool p2sh)
 {
-        /* BOLT #03:
-         * Signatures are 73 bytes long (the maximum length).
-         */
-	return 1 + /* Prefix: 4 elements to push on stack */
-		(1 + 0) + /* [0]: witness-marker-and-flag */
-		(1 + 73) + /* [1] Party A signature and length prefix */
-		(1 + 73) + /* [2] Party B signature and length prefix */
-		(1 + 1 + /* [3] length prefix and numpushes (2) */
-		 1 + 33 + /* pubkey A (with prefix) */
-		 1 + 33 + /* pubkey B (with prefix) */
-		 1 + 1 /* num sigs required and checkmultisig */
-		);
-}
-
-size_t change_weight(void)
-{
-	return bitcoin_tx_output_weight(chainparams->is_elements ? BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN : BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
-}
-
-struct amount_sat change_fee(u32 feerate_perkw,	size_t total_weight)
-{
-	struct amount_sat fee;
-
-	/* Must be able to pay for its own additional weight */
-	/* Rounding can cause off by one errors, so we do this */
-	if (!amount_sat_sub(&fee,
-			    amount_tx_fee(feerate_perkw, change_weight() + total_weight),
-			    amount_tx_fee(feerate_perkw, total_weight)))
-		abort();
-	return fee;
+	return bitcoin_tx_input_weight(p2sh,
+				       bitcoin_tx_simple_input_witness_weight());
 }
 
 struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw,
 				size_t total_weight)
 {
-	struct amount_sat fee = change_fee(feerate_perkw, total_weight);
+	size_t outweight;
+	struct amount_sat change_fee;
 
-	if (!amount_sat_sub(&excess, excess, fee))
+	/* Must be able to pay for its own additional weight */
+	outweight = bitcoin_tx_output_weight(BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN);
+
+	/* Rounding can cause off by one errors, so we do this */
+	if (!amount_sat_sub(&change_fee,
+			    amount_tx_fee(feerate_perkw, outweight + total_weight),
+			    amount_tx_fee(feerate_perkw, total_weight)))
+		return AMOUNT_SAT(0);
+
+	if (!amount_sat_sub(&excess, excess, change_fee))
 		return AMOUNT_SAT(0);
 
 	/* Must be non-dust */
@@ -993,18 +897,3 @@ struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw,
 
 	return excess;
 }
-
-u32 tx_feerate(const struct bitcoin_tx *tx)
-{
-
-	u32 feerate;
-
-	/* Fee should not overflow! */
-	if (!amount_feerate(&feerate,
-			    bitcoin_tx_compute_fee(tx),
-			    bitcoin_tx_weight(tx))) {
-		abort();
-	}
-
-	return feerate;
-}

bitcoin/tx.h

@@ -1,10 +1,9 @@
 #ifndef LIGHTNING_BITCOIN_TX_H
 #define LIGHTNING_BITCOIN_TX_H
 #include "config.h"
-#include <bitcoin/chainparams.h>
-#include <bitcoin/shadouble.h>
-#include <bitcoin/signature.h>
-#include <bitcoin/varint.h>
+#include "shadouble.h"
+#include "signature.h"
+#include "varint.h"
 #include <ccan/structeq/structeq.h>
 #include <common/amount.h>
 #include <wally_transaction.h>
@@ -14,9 +13,7 @@
 /* BIP 125: Any nsequence < 0xFFFFFFFE is replacable.
  * And bitcoind uses this value. */
 #define BITCOIN_TX_RBF_SEQUENCE 0xFFFFFFFD
-
 struct wally_psbt;
-struct ripemd160;
 
 struct bitcoin_txid {
 	struct sha256_double shad;
@@ -48,17 +45,6 @@ struct bitcoin_tx_output {
 	u8 *script;
 };
 
-enum utxotype {
-	/* Obsolete: we used to have P2SH-wrapped outputs (removed in 24.02, though can still have old UTXOs) */
-	UTXO_P2SH_P2WPKH = 1,
-	/* "bech32" addresses */
-	UTXO_P2WPKH = 2,
-	/* Used for closing addresses: implies ->close_info is non-NULL */
-	UTXO_P2WSH_FROM_CLOSE = 3,
-	/* "p2tr" addresses. */
-	UTXO_P2TR = 4,
-};
-
 struct bitcoin_tx_output *new_tx_output(const tal_t *ctx,
 					struct amount_sat amount,
 					const u8 *script);
@@ -71,7 +57,7 @@ void wally_txid(const struct wally_tx *wtx, struct bitcoin_txid *txid);
 u8 *linearize_tx(const tal_t *ctx, const struct bitcoin_tx *tx);
 u8 *linearize_wtx(const tal_t *ctx, const struct wally_tx *wtx);
 
-/* Get weight of tx in Sipa; assumes it will have witnesses! */
+/* Get weight of tx in Sipa. */
 size_t bitcoin_tx_weight(const struct bitcoin_tx *tx);
 size_t wally_tx_weight(const struct wally_tx *wtx);
 
@@ -82,26 +68,10 @@ struct bitcoin_tx *bitcoin_tx(const tal_t *ctx,
 			      varint_t input_count, varint_t output_count,
 			      u32 nlocktime);
 
-/* Make a (deep) copy */
-struct bitcoin_tx *clone_bitcoin_tx(const tal_t *ctx,
-				    const struct bitcoin_tx *tx TAKES);
-
 /* This takes a raw bitcoin tx in hex. */
 struct bitcoin_tx *bitcoin_tx_from_hex(const tal_t *ctx, const char *hex,
 				       size_t hexlen);
 
-/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
-static inline void reverse_bytes(u8 *arr, size_t len)
-{
-	unsigned int i;
-
-	for (i = 0; i < len / 2; i++) {
-		unsigned char tmp = arr[i];
-		arr[i] = arr[len - 1 - i];
-		arr[len - 1 - i] = tmp;
-	}
-}
-
 /* Parse hex string to get txid (reversed, a-la bitcoind). */
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid);
@@ -111,18 +81,12 @@ bool bitcoin_txid_to_hex(const struct bitcoin_txid *txid,
 			 char *hexstr, size_t hexstr_len);
 
 /* Create a bitcoin_tx from a psbt */
-struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx,
-					struct wally_psbt *psbt TAKES);
+struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt);
 
 /* Internal de-linearization functions. */
-/* Pull a bitcoin tx, and create a PSBT wrapper for it */
 struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx,
 				   const u8 **cursor, size_t *max);
 
-/* Pull a bitcoin tx without creating a PSBT wrapper for it */
-struct bitcoin_tx *pull_bitcoin_tx_only(const tal_t *ctx,
-					const u8 **cursor, size_t *max);
-
 /* Helper to create a wally_tx_output: make sure to wally_tx_output_free!
  * Returns NULL if amount is extreme (wally doesn't like).
  */
@@ -132,12 +96,9 @@ struct wally_tx_output *wally_tx_output(const tal_t *ctx,
 
 /* Add one output to tx. */
 int bitcoin_tx_add_output(struct bitcoin_tx *tx, const u8 *script,
-			  const u8 *wscript,
+			  u8 *wscript,
 			  struct amount_sat amount);
 
-/* Remove one output. */
-void bitcoin_tx_remove_output(struct bitcoin_tx *tx, size_t outnum);
-
 /* Set the locktime for a transaction */
 void bitcoin_tx_set_locktime(struct bitcoin_tx *tx, u32 locktime);
 
@@ -153,7 +114,7 @@ int bitcoin_tx_add_input(struct bitcoin_tx *tx,
 			 struct amount_sat amount, const u8 *scriptPubkey,
 			 const u8 *input_wscript);
 
-/* This is useful because wally uses a raw byte array for txids */
+/* This helps is useful because wally uses a raw byte array for txids */
 bool wally_tx_input_spends(const struct wally_tx_input *input,
 			   const struct bitcoin_outpoint *outpoint);
 
@@ -170,6 +131,25 @@ wally_tx_output_get_amount(const struct wally_tx_output *output);
 void bitcoin_tx_output_set_amount(struct bitcoin_tx *tx, int outnum,
 				  struct amount_sat amount);
 
+/**
+ * Helper to get the script of a script's output as a tal_arr
+ *
+ * Internally we use a `wally_tx` to represent the transaction. The script
+ * attached to a `wally_tx_output` is not a `tal_arr`, so in order to keep the
+ * comfort of being able to call `tal_bytelen` and similar on a script we just
+ * return a `tal_arr` clone of the original script.
+ */
+const u8 *bitcoin_tx_output_get_script(const tal_t *ctx, const struct bitcoin_tx *tx, int outnum);
+
+/**
+ * Helper to get the script of a script's output as a tal_arr
+ *
+ * The script attached to a `wally_tx_output` is not a `tal_arr`, so in order to keep the
+ * comfort of being able to call `tal_bytelen` and similar on a script we just
+ * return a `tal_arr` clone of the original script.
+ */
+const u8 *wally_tx_output_get_script(const tal_t *ctx,
+				     const struct wally_tx_output *output);
 /**
  * Helper to get a witness script for an output.
  */
@@ -250,47 +230,11 @@ bool elements_wtx_output_is_fee(const struct wally_tx *tx, int outnum);
  */
 bool elements_tx_output_is_fee(const struct bitcoin_tx *tx, int outnum);
 
-/** Attempt to compute the elements overhead given a base bitcoin size.
- *
- * The overhead consists of 2 empty proofs for the transaction, 6 bytes of
- * proofs per input and 35 bytes per output. In addition the explicit fee
- * output will add 9 bytes and the per output overhead as well.
- */
-static inline size_t elements_tx_overhead(const struct chainparams *chainparams,
-					  size_t incount, size_t outcount)
-{
-	size_t overhead;
-
-	if (!chainparams->is_elements)
-		return 0;
-
-	/* Each transaction has surjection and rangeproof (both empty
-	 * for us as long as we use unblinded L-BTC transactions). */
-	overhead = 2 * 4;
-	/* For elements we also need to add the fee output and the
-	 * overhead for rangeproofs into the mix. */
-	overhead += (8 + 1) * 4; /* Bitcoin style output */
-
-	/* All outputs have a bit of elements overhead (incl fee) */
-	overhead += (32 + 1 + 1 + 1) * 4 * (outcount + 1); /* Elements added fields */
-
-	/* Inputs have 6 bytes of blank proofs attached. */
-	overhead += 6 * incount;
-
-	return overhead;
-}
-
 /**
  * Calculate the fees for this transaction
  */
 struct amount_sat bitcoin_tx_compute_fee(const struct bitcoin_tx *tx);
 
-/**
- * Calculate the feerate for this transaction (in perkw)
-*/
-u32 tx_feerate(const struct bitcoin_tx *tx);
-
-
 /*
  * Calculate the fees for this transaction, given a pre-computed input balance.
  *
@@ -310,15 +254,6 @@ void towire_bitcoin_tx(u8 **pptr, const struct bitcoin_tx *tx);
 void towire_bitcoin_outpoint(u8 **pptr, const struct bitcoin_outpoint *outp);
 void fromwire_bitcoin_outpoint(const u8 **cursor, size_t *max,
 			       struct bitcoin_outpoint *outp);
-char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx);
-char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid);
-char *fmt_bitcoin_outpoint(const tal_t *ctx,
-			   const struct bitcoin_outpoint *outpoint);
-char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx);
-
-/* For want of somewhere better to define them! */
-char *fmt_sha256(const tal_t *ctx, const struct sha256 *sha256);
-char *fmt_ripemd160(const tal_t *ctx, const struct ripemd160 *ripemd160);
 
 /* Various weights of transaction parts. */
 size_t bitcoin_tx_core_weight(size_t num_inputs, size_t num_outputs);
@@ -327,43 +262,26 @@ size_t bitcoin_tx_output_weight(size_t outscript_len);
 /* Weight to push sig on stack. */
 size_t bitcoin_tx_input_sig_weight(void);
 
-/* Segwit input, but with parameter for witness weight (size).
- * witness_weight must include the varint_size() for each witness element,
- * but not the varint_size() for the number of elements. */
+/* Segwit input, but with parameter for witness weight (size) */
 size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight);
 
-/* The witness weight */
-size_t bitcoin_tx_input_witness_weight(enum utxotype utxotype);
+/* The witness weight for a simple (sig + key) input */
+size_t bitcoin_tx_simple_input_witness_weight(void);
 
-/* The witness for our 2of2 input (closing or commitment tx). */
-size_t bitcoin_tx_2of2_input_witness_weight(void);
+/* We only do segwit inputs, and we assume witness is sig + key  */
+size_t bitcoin_tx_simple_input_weight(bool p2sh);
 
 /**
- * change_weight - what's the weight of a change output?
- */
-size_t change_weight(void);
-
-/**
- * change_fee - what's the cost to add a change output to this tx?
- * @feerate_perkw: feerate.
- * @total_weight: current weight of tx.
- *
- * We pass in the total_weight of the tx (up until this point) so as
- * to avoid any off-by-one errors with rounding the change fee (down)
- */
-struct amount_sat change_fee(u32 feerate_perkw,	size_t total_weight);
-
-/**
- * change_amount - Is it worth making a change output at this feerate?
+ * change_amount - Is it worth making a P2WPKH change output at this feerate?
  * @excess: input amount we have above the tx fee and other outputs.
  * @feerate_perkw: feerate.
- * @total_weight: current weight of tx.
- *
- * Change script is P2TR for Bitcoin, P2WPKH for Elements
  *
  * If it's not worth (or possible) to make change, returns AMOUNT_SAT(0).
  * Otherwise returns the amount of the change output to add (@excess minus
- * the change_fee()).
+ * the additional fee for the change output itself).
+ *
+ * We pass in the total_weight of the tx (up until this point) so as
+ * to avoid any off-by-one errors with rounding the change fee (down)
  */
 struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw,
 				size_t total_weight);

bitcoin/tx_parts.c

@@ -10,11 +10,69 @@ static void destroy_wally_tx_input(struct wally_tx_input *in)
 	wally_tx_input_free(in);
 }
 
+static struct wally_tx_input *clone_input(const struct wally_tx_input *src)
+{
+	struct wally_tx_input *in;
+	int ret;
+
+	if (is_elements(chainparams)) {
+		ret = wally_tx_elements_input_init_alloc
+			(src->txhash, sizeof(src->txhash),
+			 src->index, src->sequence,
+			 src->script, src->script_len,
+			 src->witness,
+			 src->blinding_nonce, sizeof(src->blinding_nonce),
+			 src->entropy, sizeof(src->entropy),
+			 src->issuance_amount, src->issuance_amount_len,
+			 src->inflation_keys, src->inflation_keys_len,
+			 src->issuance_amount_rangeproof,
+			 src->issuance_amount_rangeproof_len,
+			 src->inflation_keys_rangeproof,
+			 src->inflation_keys_rangeproof_len,
+			 src->pegin_witness,
+			 &in);
+	} else {
+		ret = wally_tx_input_init_alloc(src->txhash, sizeof(src->txhash),
+						src->index, src->sequence,
+						src->script, src->script_len,
+						src->witness, &in);
+	}
+	assert(ret == WALLY_OK);
+
+	tal_add_destructor(in, destroy_wally_tx_input);
+	return in;
+}
+
 static void destroy_wally_tx_output(struct wally_tx_output *out)
 {
 	wally_tx_output_free(out);
 }
 
+static struct wally_tx_output *clone_output(const struct wally_tx_output *src)
+{
+	struct wally_tx_output *out;
+	int ret;
+
+	if (is_elements(chainparams)) {
+		ret = wally_tx_elements_output_init_alloc
+			(src->script, src->script_len,
+			 src->asset, src->asset_len,
+			 src->value, src->value_len,
+			 src->nonce, src->nonce_len,
+			 src->surjectionproof, src->surjectionproof_len,
+			 src->rangeproof, src->rangeproof_len,
+			 &out);
+	} else {
+		ret = wally_tx_output_init_alloc(src->satoshi,
+						 src->script, src->script_len,
+						 &out);
+	}
+	assert(ret == WALLY_OK);
+
+	tal_add_destructor(out, destroy_wally_tx_output);
+	return out;
+}
+
 struct tx_parts *tx_parts_from_wally_tx(const tal_t *ctx,
 					const struct wally_tx *wtx,
 					int input, int output)
@@ -29,32 +87,13 @@ struct tx_parts *tx_parts_from_wally_tx(const tal_t *ctx,
 	for (size_t i = 0; i < wtx->num_inputs; i++) {
 		if (input != -1 && input != i)
 			continue;
-		if (wally_tx_input_clone_alloc(&wtx->inputs[i],
-					       &txp->inputs[i]) != WALLY_OK)
-			abort();
-		tal_add_destructor(txp->inputs[i], destroy_wally_tx_input);
+		txp->inputs[i] = clone_input(&wtx->inputs[i]);
 	}
 
 	for (size_t i = 0; i < wtx->num_outputs; i++) {
 		if (output != -1 && output != i)
 			continue;
-		if (wally_tx_output_clone_alloc(&wtx->outputs[i],
-						&txp->outputs[i]) != WALLY_OK)
-			abort();
-		tal_add_destructor(txp->outputs[i], destroy_wally_tx_output);
-
-		/* Cheat a bit by also setting the numeric satoshi
-		 * value, otherwise we end up converting a
-		 * number of times */
-		if (chainparams->is_elements) {
-			struct amount_asset asset;
-			struct amount_sat sats;
-			asset = wally_tx_output_get_amount(txp->outputs[i]);
-			/* FIXME: non l-btc assets */
-			assert(amount_asset_is_main(&asset));
-			sats = amount_asset_to_sat(&asset);
-			txp->outputs[i]->satoshi = sats.satoshis; /* Raw: wally conversion */
-		}
+		txp->outputs[i] = clone_output(&wtx->outputs[i]);
 	}
 	tal_wally_end(txp);
 
@@ -102,7 +141,7 @@ fromwire_wally_tx_witness_stack(const tal_t *ctx,
 
 	tal_add_destructor(ws, destroy_wally_tx_witness_stack);
 out:
-	tal_wally_end_onto(ctx, ws, struct wally_tx_witness_stack);
+	tal_wally_end(tal_steal(ctx, ws));
 	return ws;
 }
 
@@ -200,7 +239,7 @@ static struct wally_tx_input *fromwire_wally_tx_input(const tal_t *ctx,
 		tal_add_destructor(in, destroy_wally_tx_input);
 	}
 
-	tal_wally_end_onto(ctx, in, struct wally_tx_input);
+	tal_wally_end(tal_steal(ctx, in));
 	return in;
 }
 
@@ -242,9 +281,6 @@ static struct wally_tx_output *fromwire_wally_tx_output(const tal_t *ctx,
 			 surjectionproof, tal_bytelen(surjectionproof),
 			 rangeproof, tal_bytelen(rangeproof),
 			 &out);
-
-		/* As a convenience, we sent the value over as satoshis */
-		out->satoshi = fromwire_u64(cursor, max);
 	} else {
 		u64 satoshi;
 		satoshi = fromwire_u64(cursor, max);
@@ -258,7 +294,7 @@ static struct wally_tx_output *fromwire_wally_tx_output(const tal_t *ctx,
 	} else {
 		tal_add_destructor(out, destroy_wally_tx_output);
 	}
-	tal_wally_end_onto(ctx, out, struct wally_tx_output);
+	tal_wally_end(tal_steal(ctx, out));
 
 	return out;
 }
@@ -312,8 +348,6 @@ static void towire_wally_tx_output(u8 **pptr, const struct wally_tx_output *out)
 				out->surjectionproof_len);
 		towire_u32(pptr, out->rangeproof_len);
 		towire_u8_array(pptr, out->rangeproof, out->rangeproof_len);
-		/* Copy the value over, as a convenience */
-		towire_u64(pptr, out->satoshi);
 	} else {
 		towire_u64(pptr, out->satoshi);
 	}

ccan/README

@@ -1,3 +1,3 @@
 CCAN imported from http://ccodearchive.net.
 
-CCAN version: init-2593-gca094039
+CCAN version: init-2520-gca7c5a9e

ccan/ccan/base64/base64.c

@@ -31,7 +31,7 @@ static int8_t sixbit_from_b64(const base64_maps_t *maps,
 	int8_t ret;
 
 	ret = maps->decode_map[(unsigned char)b64letter];
-	if (ret == (int8_t)'\xff') {
+	if (ret == (char)0xff) {
 		errno = EDOM;
 		return -1;
 	}
@@ -41,7 +41,7 @@ static int8_t sixbit_from_b64(const base64_maps_t *maps,
 
 bool base64_char_in_alphabet(const base64_maps_t *maps, const char b64char)
 {
-	return (maps->decode_map[(const unsigned char)b64char] != (signed char)'\xff');
+	return (maps->decode_map[(const unsigned char)b64char] != (char)0xff);
 }
 
 void base64_init_maps(base64_maps_t *dest, const char src[64])

ccan/ccan/base64/base64.h

@@ -116,7 +116,7 @@ ssize_t base64_decode_quartet_using_maps(const base64_maps_t *maps,
  * @note sets errno = EDOM if src contains invalid characters
  * @note sets errno = EINVAL if src is an invalid base64 tail
  */
-ssize_t base64_decode_tail_using_maps(const base64_maps_t *maps, char dest[3],
+ssize_t base64_decode_tail_using_maps(const base64_maps_t *maps, char *dest,
 				      const char *src, size_t srclen);
 
 

ccan/ccan/bitmap/bitmap.c

@@ -9,7 +9,7 @@
 #define BIT_ALIGN_DOWN(n)	((n) & ~(BITMAP_WORD_BITS - 1))
 #define BIT_ALIGN_UP(n)		BIT_ALIGN_DOWN((n) + BITMAP_WORD_BITS - 1)
 
-void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m)
+void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
 	unsigned long am = BIT_ALIGN_DOWN(m);
@@ -19,22 +19,22 @@ void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m)
 	assert(m >= n);
 
 	if (am < an) {
-		BITMAP_WORD(b, n) &= ~bitmap_bswap(headmask & tailmask);
+		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask & tailmask);
 		return;
 	}
 
 	if (an > n)
-		BITMAP_WORD(b, n) &= ~bitmap_bswap(headmask);
+		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask);
 
 	if (am > an)
-		memset(&BITMAP_WORD(b, an), 0,
+		memset(&BITMAP_WORD(bitmap, an), 0,
 		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
 
 	if (m > am)
-		BITMAP_WORD(b, m) &= ~bitmap_bswap(tailmask);
+		BITMAP_WORD(bitmap, m) &= ~bitmap_bswap(tailmask);
 }
 
-void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m)
+void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
 	unsigned long am = BIT_ALIGN_DOWN(m);
@@ -44,19 +44,19 @@ void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m)
 	assert(m >= n);
 
 	if (am < an) {
-		BITMAP_WORD(b, n) |= bitmap_bswap(headmask & tailmask);
+		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask & tailmask);
 		return;
 	}
 
 	if (an > n)
-		BITMAP_WORD(b, n) |= bitmap_bswap(headmask);
+		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask);
 
 	if (am > an)
-		memset(&BITMAP_WORD(b, an), 0xff,
+		memset(&BITMAP_WORD(bitmap, an), 0xff,
 		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
 
 	if (m > am)
-		BITMAP_WORD(b, m) |= bitmap_bswap(tailmask);
+		BITMAP_WORD(bitmap, m) |= bitmap_bswap(tailmask);
 }
 
 static int bitmap_clz(bitmap_word w)
@@ -76,7 +76,7 @@ static int bitmap_clz(bitmap_word w)
 #endif
 }
 
-unsigned long bitmap_ffs(const bitmap *b,
+unsigned long bitmap_ffs(const bitmap *bitmap,
 			 unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
@@ -87,7 +87,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	assert(m >= n);
 
 	if (am < an) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, n));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
 
 		w &= (headmask & tailmask);
 
@@ -95,7 +95,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	if (an > n) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, n));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
 
 		w &= headmask;
 
@@ -104,7 +104,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	while (an < am) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, an));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, an));
 
 		if (w)
 			return an + bitmap_clz(w);
@@ -113,7 +113,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	if (m > am) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, m));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, m));
 
 		w &= tailmask;
 

ccan/ccan/bitmap/bitmap.h

@@ -58,37 +58,37 @@ static inline bitmap_word bitmap_bswap(bitmap_word w)
 #define BITMAP_TAIL(_bm, _nbits) \
 	(BITMAP_TAILWORD(_bm, _nbits) & BITMAP_TAILBITS(_nbits))
 
-static inline void bitmap_set_bit(bitmap *b, unsigned long n)
+static inline void bitmap_set_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) |= BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) |= BITMAP_WORDBIT(n);
 }
 
-static inline void bitmap_clear_bit(bitmap *b, unsigned long n)
+static inline void bitmap_clear_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) &= ~BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) &= ~BITMAP_WORDBIT(n);
 }
 
-static inline void bitmap_change_bit(bitmap *b, unsigned long n)
+static inline void bitmap_change_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) ^= BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) ^= BITMAP_WORDBIT(n);
 }
 
-static inline bool bitmap_test_bit(const bitmap *b, unsigned long n)
+static inline bool bitmap_test_bit(const bitmap *bitmap, unsigned long n)
 {
-	return !!(BITMAP_WORD(b, n) & BITMAP_WORDBIT(n));
+	return !!(BITMAP_WORD(bitmap, n) & BITMAP_WORDBIT(n));
 }
 
-void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m);
-void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m);
+void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m);
+void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m);
 
-static inline void bitmap_zero(bitmap *b, unsigned long nbits)
+static inline void bitmap_zero(bitmap *bitmap, unsigned long nbits)
 {
-	memset(b, 0, bitmap_sizeof(nbits));
+	memset(bitmap, 0, bitmap_sizeof(nbits));
 }
 
-static inline void bitmap_fill(bitmap *b, unsigned long nbits)
+static inline void bitmap_fill(bitmap *bitmap, unsigned long nbits)
 {
-	memset(b, 0xff, bitmap_sizeof(nbits));
+	memset(bitmap, 0xff, bitmap_sizeof(nbits));
 }
 
 static inline void bitmap_copy(bitmap *dst, const bitmap *src,
@@ -161,36 +161,37 @@ static inline bool bitmap_subset(const bitmap *src1, const bitmap *src2,
 	return true;
 }
 
-static inline bool bitmap_full(const bitmap *b, unsigned long nbits)
+static inline bool bitmap_full(const bitmap *bitmap, unsigned long nbits)
 {
 	unsigned long i;
 
 	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
-		if (b[i].w != -1UL)
+		if (bitmap[i].w != -1UL)
 			return false;
 	}
 	if (BITMAP_HASTAIL(nbits) &&
-	    (BITMAP_TAIL(b, nbits) != BITMAP_TAILBITS(nbits)))
+	    (BITMAP_TAIL(bitmap, nbits) != BITMAP_TAILBITS(nbits)))
 		return false;
 
 	return true;
 }
 
-static inline bool bitmap_empty(const bitmap *b, unsigned long nbits)
+static inline bool bitmap_empty(const bitmap *bitmap, unsigned long nbits)
 {
 	unsigned long i;
 
 	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
-		if (b[i].w != 0)
+		if (bitmap[i].w != 0)
 			return false;
 	}
-	if (BITMAP_HASTAIL(nbits) && (BITMAP_TAIL(b, nbits) != 0))
+	if (BITMAP_HASTAIL(nbits) && (BITMAP_TAIL(bitmap, nbits) != 0))
 		return false;
 
 	return true;
 }
 
-unsigned long bitmap_ffs(const bitmap *b, unsigned long n, unsigned long m);
+unsigned long bitmap_ffs(const bitmap *bitmap,
+			 unsigned long n, unsigned long m);
 
 /*
  * Allocation functions
@@ -220,26 +221,26 @@ static inline bitmap *bitmap_alloc1(unsigned long nbits)
 	return bitmap;
 }
 
-static inline bitmap *bitmap_realloc0(bitmap *b,
+static inline bitmap *bitmap_realloc0(bitmap *bitmap,
 				      unsigned long obits, unsigned long nbits)
 {
-	b = realloc(b, bitmap_sizeof(nbits));
+	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
 
-	if ((nbits > obits) && b)
-		bitmap_zero_range(b, obits, nbits);
+	if ((nbits > obits) && bitmap)
+		bitmap_zero_range(bitmap, obits, nbits);
 
-	return b;
+	return bitmap;
 }
 
-static inline bitmap *bitmap_realloc1(bitmap *b,
+static inline bitmap *bitmap_realloc1(bitmap *bitmap,
 				      unsigned long obits, unsigned long nbits)
 {
-	b = realloc(b, bitmap_sizeof(nbits));
+	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
 
-	if ((nbits > obits) && b)
-		bitmap_fill_range(b, obits, nbits);
+	if ((nbits > obits) && bitmap)
+		bitmap_fill_range(bitmap, obits, nbits);
 
-	return b;
+	return bitmap;
 }
 
 #endif /* CCAN_BITMAP_H_ */

ccan/ccan/bitops/test/run.c

@@ -10,7 +10,7 @@ int main(void)
 	plan_tests(68 + 6 * (31 + 63));
 
 	for (i = 0; i < 32; i++)
-		ok1(bitops_ffs32(1U << i) == i+1);
+		ok1(bitops_ffs32(1 << i) == i+1);
 	ok1(bitops_ffs32(0) == 0);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_ffs64((uint64_t)1 << i) == i+1);
@@ -25,19 +25,19 @@ int main(void)
 	ok1(bitops_ffs64(0) == 0);
 
 	for (i = 0; i < 32; i++)
-		ok1(bitops_clz32(1U << i) == 31 - i);
+		ok1(bitops_clz32(1 << i) == 31 - i);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_clz64((uint64_t)1 << i) == 63 - i);
 
 	/* Lower bits don't effect results */
 	for (i = 0; i < 32; i++)
-		ok1(bitops_clz32((1U << i) + (1U << i)-1) == 31 - i);
+		ok1(bitops_clz32((1 << i) + (1 << i)-1) == 31 - i);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_clz64(((uint64_t)1 << i) + ((uint64_t)1 << i)-1)
 		    == 63 - i);
 
 	for (i = 0; i < 32; i++)
-		ok1(bitops_ctz32(1U << i) == i);
+		ok1(bitops_ctz32(1 << i) == i);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_ctz64((uint64_t)1 << i) == i);
 

ccan/ccan/crypto/hmac_sha256/hmac_sha256.c

@@ -35,8 +35,7 @@ void hmac_sha256_init(struct hmac_sha256_ctx *ctx,
 	 *  (e.g., if K is of length 20 bytes and B=64, then K will be
 	 *   appended with 44 zero bytes 0x00)
 	 */
-	if (ksize != 0)
-		memcpy(k_ipad, k, ksize);
+	memcpy(k_ipad, k, ksize);
 	memset((char *)k_ipad + ksize, 0, HMAC_SHA256_BLOCKSIZE - ksize);
 
 	/*

ccan/ccan/fdpass/fdpass.c

@@ -3,7 +3,6 @@
 #include <sys/socket.h>
 #include <errno.h>
 #include <string.h>
-#include <sys/types.h>
 
 bool fdpass_send(int sockout, int fd)
 {

ccan/ccan/graphql/LICENSE

@@ -1 +0,0 @@
-../../licenses/BSD-MIT

ccan/ccan/graphql/_info

@@ -1,66 +0,0 @@
-#include "config.h"
-#include <stdio.h>
-#include <string.h>
-
-/**
- * graphql - Routines to lex and parse GraphQL.
- *
- * This code contains routines to lex and parse GraphQL code.
- * This code was written per the spec at:
- *    https://spec.graphql.org/draft/
- * ...dated Fri, May 21, 2021 at the time of writing.
- *    Copyright (c) 2021 WhiteCloudFarm.org <robert.lee.dickinson@gmail.com>
- *    <https://github.com/rl-d/ccan>
- * 
- * Example:
- *	
- *	int main(int argc, char *argv[]) {
- *	
- *		const char *input_string = "{ fieldName }";
- *		struct list_head *output_tokens;
- *		struct graphql_executable_document *output_document;
- *	
- *		const char *errmsg = graphql_lexparse(
- *			NULL, // tal context
- *			input_string,
- *			&output_tokens, // variable to receive tokens
- *			&output_document); // variable to receive AST
- *	
- *		if (errmsg) {
- *			struct graphql_token *last_token;
- *			last_token = list_tail(output_tokens, struct graphql_token, node);
- *			printf("Line %d, col %d: %s",
- *				last_token->source_line,
- *				last_token->source_column + last_token->source_len,
- *				errmsg);
- *		} else {
- *			// Normally you would check every indirection in the resulting AST for null
- *			// pointers, but for simplicity of example:
- *			printf("A field from the parsed string: %s\n",
- *				output_document->first_def->op_def->sel_set->
- *				first->field->name->token_string);
- *		}
- *	
- *		output_tokens = tal_free(output_tokens);
- *	}
- *
- * License: BSD-MIT
- */
-int main(int argc, char *argv[])
-{
-	/* Expect exactly one argument */
-	if (argc != 2)
-		return 1;
-
-	if (strcmp(argv[1], "depends") == 0) {
-		printf("ccan/list\n");
-		printf("ccan/str\n");
-		printf("ccan/tal\n");
-		printf("ccan/tal/str\n");
-		printf("ccan/utf8\n");
-		return 0;
-	}
-
-	return 1;
-}
-

ccan/ccan/graphql/graphql.c

@@ -1,968 +0,0 @@
-/* MIT (BSD) license - see LICENSE file for details */
-#include "graphql.h"
-
-#include "ccan/tal/str/str.h"
-#include "ccan/utf8/utf8.h"
-
-
-/* GraphQL character classes
- *
- * These definitions are meant to reflect the GraphQL specification as
- * literally as possible.
- */
-#define SOURCE_CHAR(c) ((c) == '\t' || (c) == '\n' || (c) == '\r' || ((c) >= 32 && (c) <= 65535))
-#define WHITE_SPACE(c) ((c) == '\t' || (c) == ' ')
-#define LINE_TERMINATOR(c) ((c) == '\n' || (c) == '\r')
-#define COMMENT(c) ((c) == '#')
-#define COMMENT_CHAR(c) (SOURCE_CHAR(c) && !LINE_TERMINATOR(c))
-#define STRING_CHAR(c) (SOURCE_CHAR(c) && !LINE_TERMINATOR(c) && (c)!='"' && (c)!='\\')
-#define BLOCK_STRING_CHAR(c) (SOURCE_CHAR(c))
-#define COMMA(c) ((c) == ',')
-#define EOF_CHAR(c) ((c) == 0 || (c) == 4)
-#define PUNCTUATOR(c) (strchr("!$&().:=@[]{|}", c))
-#define HEX_DIGIT(c) (DIGIT(c) || ((c) >= 'a' && (c) <= 'f') || ((c) >= 'A' && (c) <= 'F'))
-#define DIGIT(c) ((c) >= '0' && (c) <= '9')
-#define NAME_START(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z') || (c) == '_')
-#define NAME_CONTINUE(c) (NAME_START(c) || DIGIT(c))
-
-// Helper for copying an overlapping string, since strcpy() is not safe for that
-#define cpystr(d,s) { char *cpystr_p; char *cpystr_q; for(cpystr_p = (s), cpystr_q = (d); *cpystr_p;) *cpystr_q++ = *cpystr_p++; *cpystr_q++ = *cpystr_p++; }
-
-/* Parser shorthands
- *
- * These shorthands are motivated by the parser functions, so they can be
- * written in a format that corresponds closely to the specification.
- */
-#define RET static void *
-#define PARAMS struct list_head *tokens, struct list_head *used, const char **err
-#define ARGS tokens, used, err
-#define INIT(type) \
-	struct graphql_token *rollback_top = list_top(tokens, struct graphql_token, node); \
-	struct graphql_##type *obj = talz(tokens, struct graphql_##type); \
-	(void)rollback_top; /* avoids unused variable warning */ \
-
-#define EXIT \
-	goto exit_label; /* avoids unused label warning */ \
-	exit_label: \
-	if (*err) obj = tal_free(obj); \
-	return obj; \
-
-#define CONSUME_ONE list_add(used, &list_pop(tokens, struct graphql_token, node)->node);
-#define RESTORE_ONE list_add(tokens, &list_pop(used, struct graphql_token, node)->node);
-#define ROLLBACK(args) while (list_top(tokens, struct graphql_token, node) != rollback_top) { RESTORE_ONE; }
-#define OR if (!*err) goto exit_label; *err = NULL;
-#define REQ if (*err) { ROLLBACK(args); goto exit_label; }
-#define OPT *err = NULL;
-#define WHILE_OPT while(!*err); *err = NULL;
-#define LOOKAHEAD(args, tok) struct graphql_token *tok = list_top(tokens, struct graphql_token, node);
-#define MSG(msg) if (*err) *err = msg;
-
-
-/* The following parser functions are written in a way that corresponds to the
- * grammar defined in the GraphQL specification. The code is not intended to
- * look like normal C code; it's designed for parsing clarity rather than C
- * style. Think of it as something generated rather than something to read.
- * For that reason, the functions follow special rules:
- *
- *	- The declaration is standardized with RET and PARAMS
- *	- The "err" argument is assumed to be NULL upon entrance
- *	- The "err" argument is set on failure
- *	- If the function fails to parse, then "tokens" shall be as it was upon entrance
- *	- INIT and EXIT macros are used
- *	- Macros such as REQ and OPT facilitate readability and conciseness
- */
-
-/* The following functions construct the "leaves" of the abstract syntax tree. */
-
-RET parse_keyword(PARAMS, const char *keyword, const char *errmsg) {
-	struct graphql_token *tok = list_top(tokens, struct graphql_token, node);
-	if (!tok || tok->token_type != 'a') {
-		*err = errmsg; return NULL;
-	}
-	if (!streq(tok->token_string, keyword)) {
-		*err = errmsg; return NULL;
-	}
-	CONSUME_ONE;
-	return tok;
-}
-
-// Note: a static buffer is used here.
-RET parse_punct(PARAMS, int punct) {
-	static char punctbuf[16];
-	struct graphql_token *tok = list_top(tokens, struct graphql_token, node);
-	if (!tok || tok->token_type != punct) {
-		if (punct == PUNCT_SPREAD)
-			sprintf(punctbuf, "expected: '...'");
-		else
-			sprintf(punctbuf, "expected: '%c'", punct);
-		*err = punctbuf; return NULL;
-	}
-	CONSUME_ONE;
-	return tok;
-}
-
-RET parse_name(PARAMS) {
-	struct graphql_token *tok = list_top(tokens, struct graphql_token, node);
-	if (!tok || tok->token_type != 'a') {
-		*err = "name expected"; return NULL;
-	}
-	CONSUME_ONE;
-	return tok;
-}
-
-RET parse_int(PARAMS) {
-	struct graphql_token *tok = list_top(tokens, struct graphql_token, node);
-	if (!tok || tok->token_type != 'i') {
-		*err = "integer expected"; return NULL;
-	}
-	CONSUME_ONE;
-	return tok;
-}
-
-RET parse_float(PARAMS) {
-	struct graphql_token *tok = list_top(tokens, struct graphql_token, node);
-	if (!tok || tok->token_type != 'f') {
-		*err = "float expected"; return NULL;
-	}
-	CONSUME_ONE;
-	return tok;
-}
-
-RET parse_string(PARAMS) {
-	struct graphql_token *tok = list_top(tokens, struct graphql_token, node);
-	if (!tok || tok->token_type != 's') {
-		*err = "string expected"; return NULL;
-	}
-	CONSUME_ONE;
-	return tok;
-}
-
-// The following functions create the branches of the AST.
-
-/*
-RET parse_non_null_type_2(PARAMS) {
-	INIT(non_null_type);
-	parse_list_type(ARGS); REQ;
-	parse_punct(ARGS, '!'); REQ;
-	EXIT;
-}
-
-RET parse_non_null_type_1(PARAMS) {
-	INIT(non_null_type);
-	parse_named_type(ARGS); REQ;
-	parse_punct(ARGS, '!'); REQ;
-	EXIT;
-}
-
-RET parse_non_null_type(PARAMS) {
-	INIT(non_null_type);
-	parse_non_null_type_1(ARGS); OR
-	parse_non_null_type_2(ARGS);
-	EXIT;
-}
-
-RET parse_list_type(PARAMS) {
-	INIT(list_type);
-	parse_punct(ARGS, '['); REQ
-	parse_type(ARGS); REQ
-	parse_punct(ARGS, ']'); REQ
-	EXIT;
-}
-*/
-
-RET parse_named_type(PARAMS) {
-	INIT(named_type);
-	obj->name = parse_name(ARGS);
-	EXIT;
-}
-
-RET parse_type(PARAMS) {
-	INIT(type);
-	obj->named = parse_named_type(ARGS);
-/*
-	OR
-	obj->list = parse_list_type(ARGS); OR
-	obj->non_null = parse_non_null_type(ARGS);
-*/
-	EXIT;
-}
-
-RET parse_variable(PARAMS) {
-	INIT(variable);
-	parse_punct(ARGS, '$'); REQ
-	obj->name = parse_name(ARGS); REQ
-	EXIT;
-}
-
-RET parse_value(PARAMS);
-
-RET parse_list_value(PARAMS) {
-	INIT(list_value);
-	parse_punct(ARGS, '['); REQ
-	parse_punct(ARGS, ']');
-	while (*err) {
-		*err = NULL;
-		parse_value(ARGS); MSG("expected: value or ']'"); REQ
-		parse_punct(ARGS, ']');
-	}
-	EXIT;
-}
-
-RET parse_enum_value(PARAMS) {
-	INIT(enum_value);
-	obj->val = parse_name(ARGS); REQ
-	struct graphql_token *tok = list_top(used, struct graphql_token, node);
-	if (streq(tok->token_string, "true")
-	 || streq(tok->token_string, "false")
-	 || streq(tok->token_string, "null")) {
-		*err = "enum value cannot be true, false, or null";
-		ROLLBACK(ARGS);
-	}
-	EXIT;
-}
-
-RET parse_null_value(PARAMS) {
-	INIT(null_value);
-	obj->val = parse_keyword(ARGS, "null", "null expected");
-	EXIT;
-}
-
-RET parse_string_value(PARAMS) {
-	INIT(string_value);
-	obj->val = parse_string(ARGS);
-	EXIT;
-}
-
-RET parse_boolean_value(PARAMS) {
-	INIT(boolean_value);
-	obj->val = parse_keyword(ARGS, "true", "invalid boolean value"); OR
-	obj->val = parse_keyword(ARGS, "false", "invalid boolean value");
-	EXIT;
-}
-
-RET parse_float_value(PARAMS) {
-	INIT(float_value);
-	obj->val = parse_float(ARGS);
-	EXIT;
-}
-
-RET parse_int_value(PARAMS) {
-	INIT(int_value);
-	obj->val = parse_int(ARGS);
-	EXIT;
-}
-
-RET parse_object_field(PARAMS) {
-	INIT(object_field);
-	obj->name = parse_name(ARGS); REQ
-	parse_punct(ARGS, ':'); REQ
-	obj->val = parse_value(ARGS); REQ
-	EXIT;
-}
-
-RET parse_object_value(PARAMS) {
-	INIT(object_value);
-	parse_punct(ARGS, '{'); REQ
-	parse_punct(ARGS, '}');
-	struct graphql_object_field *p = NULL;
-	while (*err) {
-		*err = NULL;
-		if (!p) {
-			obj->first = p = parse_object_field(ARGS); MSG("expected: object field or '}'"); REQ
-		} else {
-			p->next = parse_object_field(ARGS); MSG("expected: object field or '}'"); REQ
-			p = p->next;
-		}
-		parse_punct(ARGS, '}');
-	}
-	EXIT;
-}
-
-RET parse_default_value(PARAMS) {
-	INIT(default_value);
-	parse_punct(ARGS, '='); REQ
-	obj->val = parse_value(ARGS); REQ
-	EXIT;
-}
-
-RET parse_value(PARAMS) {
-	INIT(value);
-	obj->var = parse_variable(ARGS); // FIXME: if not const
-	OR
-	obj->int_val = parse_int_value(ARGS); OR
-	obj->float_val = parse_float_value(ARGS); OR
-	obj->str_val = parse_string_value(ARGS); OR
-	obj->bool_val = parse_boolean_value(ARGS); OR
-	obj->null_val = parse_null_value(ARGS); OR
-	obj->enum_val = parse_enum_value(ARGS); OR
-	obj->list_val = parse_list_value(ARGS); OR
-	obj->obj_val = parse_object_value(ARGS);
-	EXIT;
-}
-
-RET parse_type_condition(PARAMS) {
-	INIT(type_condition);
-	parse_keyword(ARGS, "on", "expected: 'on'"); REQ
-	obj->named_type = parse_named_type(ARGS); REQ
-	EXIT;
-}
-
-RET parse_fragment_name(PARAMS) {
-	INIT(fragment_name);
-	obj->name = parse_name(ARGS); REQ
-	struct graphql_token *tok = list_top(used, struct graphql_token, node);
-	if (streq(tok->token_string, "on")) {
-		*err = "invalid fragment name";
-		ROLLBACK(ARGS);
-	}
-	EXIT;
-}
-
-RET parse_alias(PARAMS) {
-	INIT(alias);
-	obj->name = parse_name(ARGS); REQ
-	parse_punct(ARGS, ':'); REQ
-	EXIT;
-}
-
-RET parse_argument(PARAMS) {
-	INIT(argument);
-	obj->name = parse_name(ARGS); REQ
-	parse_punct(ARGS, ':'); REQ
-	obj->val = parse_value(ARGS); REQ
-	EXIT;
-}
-
-RET parse_arguments(PARAMS) {
-	INIT(arguments);
-	parse_punct(ARGS, '('); REQ
-	obj->first = parse_argument(ARGS); REQ
-	struct graphql_argument *p = obj->first;
-	parse_punct(ARGS, ')');
-	while (*err) {
-		*err = NULL;
-		p->next = parse_argument(ARGS); MSG("expected: argument or ')'"); REQ;
-		p = p->next;
-		parse_punct(ARGS, ')');
-	}
-	EXIT;
-}
-
-RET parse_directive(PARAMS) {
-	INIT(directive);
-	parse_punct(ARGS, '@'); REQ
-	obj->name = parse_name(ARGS); REQ
-	obj->args = parse_arguments(ARGS); OPT
-	EXIT;
-}
-
-RET parse_directives(PARAMS) {
-	INIT(directives);
-	obj->first = parse_directive(ARGS); REQ
-	struct graphql_directive *p = obj->first;
-	do {
-		p->next = parse_directive(ARGS);
-		p = p->next;
-	} WHILE_OPT;
-	EXIT;
-}
-
-RET parse_fragment_spread(PARAMS) {
-	INIT(fragment_spread);
-	parse_punct(ARGS, PUNCT_SPREAD); REQ
-	obj->name = parse_fragment_name(ARGS); REQ
-	obj->directives = parse_directives(ARGS); OPT
-	EXIT;
-}
-
-RET parse_variable_definition(PARAMS) {
-	INIT(variable_definition);
-	obj->var = parse_variable(ARGS); REQ
-	parse_punct(ARGS, ':'); REQ
-	obj->type = parse_type(ARGS); REQ
-	obj->default_val = parse_default_value(ARGS); OPT
-	obj->directives = parse_directives(ARGS); OPT
-	EXIT;
-}
-
-RET parse_variable_definitions(PARAMS) {
-	INIT(variable_definitions);
-	parse_punct(ARGS, '('); REQ
-	obj->first = parse_variable_definition(ARGS); REQ
-	struct graphql_variable_definition *p = obj->first;
-	parse_punct(ARGS, ')');
-	while (*err) {
-		*err = NULL;
-		p->next = parse_variable_definition(ARGS); MSG("expected: variable definition or ')'"); REQ
-		p = p->next;
-		parse_punct(ARGS, ')');
-	}
-	EXIT;
-}
-
-RET parse_selection_set(PARAMS);
-
-RET parse_fragment_definition(PARAMS) {
-	INIT(fragment_definition);
-	parse_keyword(ARGS, "fragment", "fragment expected"); REQ
-	obj->name = parse_fragment_name(ARGS); REQ
-	obj->type_cond = parse_type_condition(ARGS); REQ
-	obj->directives = parse_directives(ARGS); OPT
-	obj->sel_set = parse_selection_set(ARGS); REQ
-	EXIT;
-}
-
-RET parse_inline_fragment(PARAMS) {
-	INIT(inline_fragment);
-	parse_punct(ARGS, PUNCT_SPREAD); REQ
-	obj->type_cond = parse_type_condition(ARGS); OPT
-	obj->directives = parse_directives(ARGS); OPT
-	obj->sel_set = parse_selection_set(ARGS); REQ
-	EXIT;
-}
-
-RET parse_field(PARAMS) {
-	INIT(field);
-	obj->alias = parse_alias(ARGS); OPT
-	obj->name = parse_name(ARGS); REQ
-	obj->args = parse_arguments(ARGS); OPT
-	obj->directives = parse_directives(ARGS); OPT
-	obj->sel_set = parse_selection_set(ARGS); OPT
-	EXIT;
-}
-
-RET parse_selection(PARAMS) {
-	INIT(selection);
-	obj->field = parse_field(ARGS); OR
-	obj->frag_spread = parse_fragment_spread(ARGS); OR
-	obj->inline_frag = parse_inline_fragment(ARGS);
-	MSG("expected: field, fragment spread, or inline fragment");
-	EXIT;
-}
-
-RET parse_selection_set(PARAMS) {
-	INIT(selection_set);
-	parse_punct(ARGS, '{'); REQ;
-	obj->first = parse_selection(ARGS); REQ;
-	struct graphql_selection *p = obj->first;
-	parse_punct(ARGS, '}');
-	while (*err) {
-		*err = NULL;
-		p->next = parse_selection(ARGS); MSG("expected: selection or '}'"); REQ;
-		p = p->next;
-		parse_punct(ARGS, '}');
-	}
-	EXIT;
-}
-
-RET parse_operation_type(PARAMS) {
-	INIT(operation_type);
-	const char *errmsg = "expected: query, mutation, or subscription";
-	obj->op_type = parse_keyword(ARGS, "query", errmsg); OR
-	obj->op_type = parse_keyword(ARGS, "mutation", errmsg); OR
-	obj->op_type = parse_keyword(ARGS, "subscription", errmsg);
-	EXIT;
-}
-
-RET parse_operation_definition(PARAMS) {
-	INIT(operation_definition);
-	obj->op_type = parse_operation_type(ARGS);
-	if (!*err) {
-		obj->op_name = parse_name(ARGS); OPT
-		obj->vars = parse_variable_definitions(ARGS); OPT
-		obj->directives = parse_directives(ARGS); OPT
-	} else
-		*err = NULL;
-	obj->sel_set = parse_selection_set(ARGS);
-	if (*err) ROLLBACK(ARGS);
-	EXIT;
-}
-
-RET parse_executable_definition(PARAMS) {
-	INIT(executable_definition);
-	obj->op_def = parse_operation_definition(ARGS); MSG("invalid operation or fragment definition"); OR
-	obj->frag_def = parse_fragment_definition(ARGS); MSG("invalid operation or fragment definition"); 
-	EXIT;
-}
-
-RET parse_executable_document(PARAMS) {
-	INIT(executable_document);
-	obj->first_def = parse_executable_definition(ARGS); REQ
-	struct graphql_executable_definition *p = obj->first_def;
-	do {
-		p->next_def = parse_executable_definition(ARGS);
-		p = p->next_def;
-	} WHILE_OPT;
-	EXIT;
-}
-
-RET parse_definition(PARAMS) {
-	INIT(definition);
-	obj->executable_def = parse_executable_definition(ARGS);
-/*	OR
-	obj->type_system_def = parse_type_system_definition_or_extension(ARGS);
-	// NOTE: Optional type system is not (yet) implemented.
-*/
-	EXIT;
-}
-
-RET parse_document(PARAMS) {
-	INIT(document);
-	obj->first_def = parse_definition(ARGS); REQ
-	struct graphql_definition *p = obj->first_def;
-	do {
-		p->next_def = parse_definition(ARGS);
-		p = p->next_def;
-	} WHILE_OPT;
-	EXIT;
-}
-void *currently_unused = parse_document; // to hide the warning till this is used
-
-/* Convert input string into tokens.
- *
- * All data (i.e. the list and the tokens it contains) are allocated to the
- * specified tal context.
- */
-const char *graphql_lex(const tal_t *ctx, const char *input, struct list_head **tokens) {
-
-	unsigned int c;
-	const char *p, *line_beginning;
-	unsigned int line_num = 1;
-	struct list_head *tok_list;
-	struct graphql_token *tok;
-
-	// Initialize token output list.
-	tok_list = tal(ctx, struct list_head);
-	if (tokens)
-		*tokens = tok_list;
-	list_head_init(tok_list);
-
-	// Note: label and goto are used here like a continue statement except that
-        // it skips iteration, for when characters are fetched in the loop body.
-	p = input;
-	line_beginning = p;
-	do {
-		c = *p++;
-newchar:
-		// Consume line terminators and increment line counter.
-		if (LINE_TERMINATOR(c)) {
-			unsigned int c0 = c;
-			c = *p++;
-			if (c0 == 10 || c0 == 13)
-				line_num++;
-			if (c0 == 13 && c == 10)
-				c = *p++;
-			line_beginning = p - 1;
-			goto newchar;
-		}
-
-		// Consume other ignored tokens.
-		if (COMMA(c) || WHITE_SPACE(c)) {
-			c = *p++;
-			goto newchar;
-		}
-		if (COMMENT(c)) {
-			while (!EOF_CHAR(c) && COMMENT_CHAR(c))
-				c = *p++;
-			goto newchar;
-		}
-
-		// Return success when end is reached.
-		if (EOF_CHAR(c))
-			return GRAPHQL_SUCCESS;
-
-		// Punctuator tokens.
-		if (PUNCTUATOR(c)) {
-
-			// Note beginning of token in input.
-			const char *start = p - 1;
-
-			// Handle the ... multi-character case.
-			if (c == '.') {
-				c = *p++;
-				if (c != '.')
-					return "unrecognized punctuator";
-				c = *p++;
-				if (c != '.')
-					return "unrecognized punctuator";
-				c = PUNCT_SPREAD;
-			}
-
-			tok = talz(tok_list, struct graphql_token);
-			list_add_tail(tok_list, &tok->node);
-			tok->token_type = c;
-			tok->token_string = NULL;
-			tok->source_line = line_num;
-			tok->source_column = start - line_beginning + 1;
-			tok->source_offset = start - input;
-			tok->source_len = p - start;
-
-		} else if (NAME_START(c)) {
-
-			// Name/identifier tokens.
-			tok = talz(tok_list, struct graphql_token);
-			list_add_tail(tok_list, &tok->node);
-			tok->token_type = 'a';
-			// tok->token_string updated below.
-			tok->source_line = line_num;
-			tok->source_column = p - line_beginning;
-			// tok->source_len updated below.
-
-			// Note the beginning of the name.
-			const char *name_begin = p - 1;
-			const char *name_end;
-			int name_len;
-
-			// Consume the rest of the token.
-			do {
-				c = *p++;
-			} while (NAME_CONTINUE(c));
-
-			// Note the end of the name and calculate the length.
-			name_end = p - 1;
-			name_len = name_end - name_begin;
-			tok->source_offset = name_begin - input;
-			tok->source_len = name_len;
-
-			// Copy the token string.
-			tok->token_string = tal_strndup(tok, name_begin, name_len);
-
-			goto newchar;
-
-		} else if (DIGIT(c) || c == '-') {
-
-			// Number tokens.
-			const char *num_start = p - 1;
-			char type = 'i';
-
-			if (c == '-') {
-				c = *p++;
-				if (!DIGIT(c))
-					return "negative sign must precede a number";
-			}
-
-			if (c == '0') {
-				c = *p++;
-				if (DIGIT(c))
-					return "leading zeros are not allowed";
-			} else {
-				do {
-					c = *p++;
-				} while(DIGIT(c));
-			}
-
-			if (c == '.') {
-				type = 'f';
-				if (!DIGIT(*p))
-					return "invalid float value fractional part";
-				do {
-					c = *p++;
-				} while(DIGIT(c));
-			}
-
-			if (c == 'e' || c == 'E') {
-				type = 'f';
-				c = *p++;
-				if (c == '+' || c == '-')
-					c = *p++;
-				if (!DIGIT(*p))
-					return "invalid float value exponent part";
-				do {
-					c = *p++;
-				} while(DIGIT(c));
-			}
-
-			if (c == '.' || NAME_START(c))
-				return "invalid numeric value";
-
-			const char *num_end = p - 1;
-			int num_len = num_end - num_start;
-
-			tok = talz(tok_list, struct graphql_token);
-			list_add_tail(tok_list, &tok->node);
-			tok->token_type = type;
-			tok->token_string = tal_strndup(tok, num_start, num_len);
-			tok->source_line = line_num;
-			tok->source_column = num_start - line_beginning + 1;
-			tok->source_offset = num_start - input;
-			tok->source_len = num_len;
-
-			goto newchar;
-
-		} else if (c == '"') {
-
-			// String tokens.
-			c = *p++;
-			const char *str_begin = p - 1;
-			const char *str_end;
-			bool str_block = false;
-			if (c == '"') {
-				c = *p++;
-				if (c == '"') {
-					// block string
-					str_block = true;
-					str_begin += 2;
-					int quotes = 0;
-					do {
-						c = *p++;
-						if (c == '\"') quotes++; else quotes = 0;
-						if (quotes == 3 && *(p-4) == '\\') quotes = 0;
-					} while (BLOCK_STRING_CHAR(c) && quotes < 3);
-					if (quotes == 3) {
-						c = *--p;
-						c = *--p;
-					}
-					str_end = p - 1;
-					if (c != '"')
-						return "unterminated string or invalid character";
-					c = *p++;
-					if (c != '"')
-						return "invalid string termination";
-					c = *p++;
-					if (c != '"')
-						return "invalid string termination";
-				} else {
-					// empty string
-					str_end = str_begin;
-					--p;
-				}
-			} else {
-				// normal string
-				--p;
-				do {
-					c = *p++;
-					if (c == '\\') {
-						c = *p++;
-						if (strchr("\"\\/bfnrtu", c)) {
-							if (c == 'u') {
-								c = *p++;
-								if (!HEX_DIGIT(c))
-									return "invalid unicode escape sequence";
-								c = *p++;
-								if (!HEX_DIGIT(c))
-									return "invalid unicode escape sequence";
-								c = *p++;
-								if (!HEX_DIGIT(c))
-									return "invalid unicode escape sequence";
-								c = *p++;
-								if (!HEX_DIGIT(c))
-									return "invalid unicode escape sequence";
-							} else {
-								c = 'a'; // anything besides a quote to let the loop continue
-							}
-						} else {
-							return "invalid string escape sequence";
-						}
-					}
-				} while (STRING_CHAR(c));
-				if (c != '"')
-					return "unterminated string or invalid character";
-				str_end = p - 1;
-			}
-			int str_len = str_end - str_begin;
-
-			tok = talz(tok_list, struct graphql_token);
-			list_add_tail(tok_list, &tok->node);
-			tok->token_type = 's';
-			tok->token_string = tal_strndup(tok, str_begin, str_len);
-			tok->source_line = line_num;
-			tok->source_column = str_begin - line_beginning + 1;
-			tok->source_offset = str_begin - input;
-			tok->source_len = str_len;
-
-			// Process escape sequences. These always shorten the string (so the memory allocation is always enough).
-			char d;
-			char *q = tok->token_string;
-			char *rewrite_dest;
-			int quotes = 0;
-			while ((d = *q++)) {
-				if (str_block) {
-					if (d == '\"') quotes++; else quotes = 0;
-					if (quotes == 3 && *(q-4) == '\\') {
-						quotes = 0;
-						rewrite_dest = q - 4;
-						cpystr(rewrite_dest, q - 3);
-					}
-				} else {
-					if (d == '\\') {
-						rewrite_dest = q - 1;
-						d = *q++;
-						switch (d) {
-						case '\"':
-							*rewrite_dest++ = '\"';
-							cpystr(rewrite_dest, q--);
-							break;
-						case 'b':
-							*rewrite_dest++ = '\b';
-							cpystr(rewrite_dest, q--);
-							break;
-						case 'f':
-							*rewrite_dest++ = '\f';
-							cpystr(rewrite_dest, q--);
-							break;
-						case 'n':
-							*rewrite_dest++ = '\n';
-							cpystr(rewrite_dest, q--);
-							break;
-						case 'r':
-							*rewrite_dest++ = '\r';
-							cpystr(rewrite_dest, q--);
-							break;
-						case 't':
-							*rewrite_dest++ = '\t';
-							cpystr(rewrite_dest, q--);
-							break;
-						case 'u': {
-								// Insert escaped character using UTF-8 multi-byte encoding.
-								char buf[5], *b = buf;
-								for (int i = 0; i < 4; i++)
-									*b++ = *q++;
-								*b = 0;
-								int code_point = strtol(buf, 0, 16);
-								int bytes = utf8_encode(code_point, rewrite_dest);
-								// note: if bytes == 0
-								// due to encoding failure,
-								// the following will safely
-								// eliminate the invalid char.
-								rewrite_dest += bytes;
-								cpystr(rewrite_dest, q--);
-							}
-							break;
-						default:
-							cpystr(rewrite_dest, --q);
-						}
-					}
-				}
-			}
-			if (str_block) {
-				// Strip leading lines.
-				q = tok->token_string;
-				for (;;) {
-					d = *q++;
-					while (WHITE_SPACE(d))
-						d = *q++;
-					if (LINE_TERMINATOR(d)) {
-						while (LINE_TERMINATOR(d))
-							d = *q++;
-						cpystr(tok->token_string, q - 1);
-						q = tok->token_string;
-					} else
-						break;
-				}
-
-				// Strip trailing lines.
-				q = tok->token_string + strlen(tok->token_string);
-				for (;;) {
-					d = *--q;
-					while (WHITE_SPACE(d))
-						d = *--q;
-					if (LINE_TERMINATOR(d)) {
-						while (LINE_TERMINATOR(d))
-							d = *--q;
-						*++q = 0;
-					} else
-						break;
-				}
-
-				// Look for common indentation.
-				char *this_indent_start;
-				const char *this_indent_end;
-				const char *common_indent_start = NULL;
-				const char *common_indent_end = common_indent_start;
-				const char *r;
-				q = tok->token_string;
-				do {
-					d = *q++;
-					this_indent_start = q - 1;
-					while (WHITE_SPACE(d))
-						d = *q++;
-					this_indent_end = q - 1;
-					if (LINE_TERMINATOR(d)) {
-						while (LINE_TERMINATOR(d))
-							d = *q++;
-						continue;
-					}
-					if (EOF_CHAR(d))
-						continue;
-
-					if (common_indent_start == NULL) {
-						common_indent_start = this_indent_start;
-						common_indent_end = this_indent_end;
-					}
-					for (r = this_indent_start; r < this_indent_end && (r - this_indent_start + common_indent_start < common_indent_end); r++) {
-						if (*r != *(r - this_indent_start + common_indent_start))
-							break;
-					}
-					common_indent_end = r - this_indent_start + common_indent_start;
-
-					while (!LINE_TERMINATOR(d) && !EOF_CHAR(d))
-						d = *q++;
-					while (LINE_TERMINATOR(d))
-						d = *q++;
-					--q;
-
-				} while (d);
-
-				// Remove common indentation.
-				int common_indent_len = common_indent_end - common_indent_start;
-				if (common_indent_len > 0) {
-					q = tok->token_string;
-					do {
-						d = *q++;
-						this_indent_start = q - 1;
-						while (WHITE_SPACE(d))
-							d = *q++;
-						this_indent_end = q - 1;
-						if (LINE_TERMINATOR(d)) {
-							while (LINE_TERMINATOR(d))
-								d = *q++;
-							continue;
-						}
-						if (EOF_CHAR(d))
-							continue;
-
-						while (!LINE_TERMINATOR(d) && !EOF_CHAR(d))
-							d = *q++;
-						--q;
-
-						cpystr(this_indent_start, this_indent_start + common_indent_len);
-						q -= common_indent_len;
-						d = *q++;
-
-						while (LINE_TERMINATOR(d))
-							d = *q++;
-						--q;
-
-					} while (d);
-				}
-			}
-			c = *p++;
-			goto newchar;
-
-		} else {
-			return "invalid source character encountered";
-		}
-
-	} while (!EOF_CHAR(c));
-
-	return "unexpected end-of-input encountered";
-}
-
-// Convert lexed tokens into AST.
-const char *graphql_parse(struct list_head *tokens, struct graphql_executable_document **doc) {
-	struct list_head used = LIST_HEAD_INIT(used);
-	const char *err = NULL;
-	*doc = parse_executable_document(tokens, &used, &err);
-	return err;
-}
-
-// Convert input string into AST.
-const char *graphql_lexparse(const tal_t *ctx, const char *input, struct list_head **tokens, struct graphql_executable_document **doc) {
-	const char *err = graphql_lex(ctx, input, tokens);
-	if (!err)
-		err = graphql_parse(*tokens, doc);
-	return err;
-}
-
-
-

ccan/ccan/graphql/graphql.h

@@ -1,291 +0,0 @@
-/* MIT (BSD) license - see LICENSE file for details */
-#ifndef __GRAPHQL_H__
-#define __GRAPHQL_H__ 1
-
-#include <stdio.h>
-
-#include <ccan/list/list.h>
-#include <ccan/tal/tal.h>
-
-// Coding constants
-#define GRAPHQL_SUCCESS ((const char *)NULL)
-
-// The following structures constitute the AST returned by the parser.
-
-struct graphql_directive {
-	struct graphql_directive *next;
-	struct graphql_token *name;
-	struct graphql_arguments *args;
-	void *data; // for application use
-};
-
-struct graphql_directives {
-	struct graphql_directive *first;
-	void *data; // for application use
-};
-
-struct graphql_named_type {
-	struct graphql_token *name;
-	void *data; // for application use
-};
-
-struct graphql_type {
-	struct graphql_named_type *named;
-//	struct graphql_list_type *list;
-//	struct graphql_non_null_type *non_null;
-	void *data; // for application use
-};
-
-struct graphql_default_value {
-	struct graphql_value *val;
-	void *data; // for application use
-};
-
-struct graphql_variable_definition {
-	struct graphql_variable_definition *next;
-	struct graphql_variable *var;
-	struct graphql_type *type;
-	struct graphql_default_value *default_val;
-	struct graphql_directives *directives;
-	void *data; // for application use
-};
-
-struct graphql_variable_definitions {
-	struct graphql_variable_definition *first;
-	void *data; // for application use
-};
-
-struct graphql_variable {
-	struct graphql_token *name;
-	void *data; // for application use
-};
-
-struct graphql_object_field {
-	struct graphql_object_field *next;
-	struct graphql_token *name;
-	struct graphql_value *val;
-	void *data; // for application use
-};
-
-struct graphql_object_value {
-	struct graphql_object_field *first;
-	void *data; // for application use
-};
-
-struct graphql_list_value {
-	struct graphql_token *val;
-	void *data; // for application use
-};
-
-struct graphql_enum_value {
-	struct graphql_token *val;
-	void *data; // for application use
-};
-
-struct graphql_null_value {
-	struct graphql_token *val;
-	void *data; // for application use
-};
-
-struct graphql_string_value {
-	struct graphql_token *val;
-	void *data; // for application use
-};
-
-struct graphql_boolean_value {
-	struct graphql_token *val;
-	void *data; // for application use
-};
-
-struct graphql_float_value {
-	struct graphql_token *val;
-	void *data; // for application use
-};
-
-struct graphql_int_value {
-	struct graphql_token *val;
-	void *data; // for application use
-};
-
-struct graphql_value {
-	struct graphql_variable *var;
-	struct graphql_int_value *int_val;
-	struct graphql_float_value *float_val;
-	struct graphql_boolean_value *bool_val;
-	struct graphql_string_value *str_val;
-	struct graphql_null_value *null_val;
-	struct graphql_enum_value *enum_val;
-	struct graphql_list_value *list_val;
-	struct graphql_object_value *obj_val;
-	void *data; // for application use
-};
-
-struct graphql_inline_fragment {
-	struct graphql_type_condition *type_cond;
-	struct graphql_directives *directives;
-	struct graphql_selection_set *sel_set;
-	void *data; // for application use
-};
-
-struct graphql_type_condition {
-	struct graphql_named_type *named_type;
-	void *data; // for application use
-};
-
-struct graphql_fragment_name {
-	struct graphql_token *name;
-	void *data; // for application use
-};
-
-struct graphql_fragment_definition {
-	struct graphql_fragment_name *name;
-	struct graphql_type_condition *type_cond;
-	struct graphql_directives *directives;
-	struct graphql_selection_set *sel_set;
-	void *data; // for application use
-};
-
-struct graphql_fragment_spread {
-	struct graphql_fragment_name *name;
-	struct graphql_directives *directives;
-	void *data; // for application use
-};
-
-struct graphql_alias {
-	struct graphql_token *name;
-	void *data; // for application use
-};
-
-struct graphql_argument {
-	struct graphql_argument *next;
-	struct graphql_token *name;
-	struct graphql_value *val;
-	void *data; // for application use
-};
-
-struct graphql_arguments {
-	struct graphql_argument *first;
-	void *data; // for application use
-};
-
-struct graphql_field {
-	struct graphql_alias *alias;
-	struct graphql_token *name;
-	struct graphql_arguments *args;
-	struct graphql_directives *directives;
-	struct graphql_selection_set *sel_set;
-	void *data; // for application use
-};
-
-struct graphql_selection {
-	struct graphql_selection *next;
-	struct graphql_field *field;
-	struct graphql_fragment_spread *frag_spread;
-	struct graphql_inline_fragment *inline_frag;
-	void *data; // for application use
-};
-
-struct graphql_selection_set {
-	struct graphql_selection *first;
-	void *data; // for application use
-};
-
-struct graphql_operation_type {
-	struct graphql_token *op_type;
-	void *data; // for application use
-};
-
-struct graphql_operation_definition {
-	struct graphql_operation_type *op_type;
-	struct graphql_token *op_name;
-	struct graphql_variable_definitions *vars;
-	struct graphql_directives *directives;
-	struct graphql_selection_set *sel_set;
-	void *data; // for application use
-};
-
-struct graphql_executable_definition {
-	struct graphql_executable_definition *next_def;
-	struct graphql_operation_definition *op_def;
-	struct graphql_fragment_definition *frag_def;
-	void *data; // for application use
-};
-
-struct graphql_executable_document {
-	struct graphql_executable_definition *first_def;
-	void *data; // for application use
-};
-
-struct graphql_definition {
-	struct graphql_definition *next_def;
-	struct graphql_executable_definition *executable_def;
-	struct graphql_type_system_definition_or_extension *type_system_def;
-	void *data; // for application use
-};
-
-struct graphql_document {
-	struct graphql_definition *first_def;
-	void *data; // for application use
-};
-
-enum token_type_enum {
-	NAME		= 'a',
-	INTEGER		= 'i',
-	FLOAT		= 'f',
-	STRING		= 's',
-	PUNCT_BANG	= '!',
-	PUNCT_SH__	= '$',
-	PUNCT_AMP	= '&',
-	PUNCT_LPAR	= '(',
-	PUNCT_RPAR	= ')',
-	PUNCT_COLON	= ':',
-	PUNCT_EQ	= '=',
-	PUNCT_AT	= '@',
-	PUNCT_LBRACKET	= '[',
-	PUNCT_RBRACKET	= ']',
-	PUNCT_LBRACE	= '{',
-	PUNCT_PIPE	= '|',
-	PUNCT_RBRACE	= '}',
-	PUNCT_SPREAD	= 0x2026, // spread operator (triple dot)
-};
-
-struct graphql_token {
-	struct list_node node;
-	enum token_type_enum token_type;
-	char *token_string;
-	unsigned int source_line;
-	unsigned int source_column;
-	unsigned int source_offset;
-	unsigned int source_len;
-	void *data; // for application use
-};
-
-/* The lexer.
- * INPUTS:
- *	input - string to parse
- *	ctx - parent tal context or NULL
- *	tokens - a variable to receive the resulting token list
- * RETURN:
- *	GRAPHQL_SUCCESS or an error string.
- */
-const char *graphql_lex(const tal_t *ctx, const char *input, struct list_head **tokens);
-
-/* The parser.
- * INPUTS:
- *	tokens - the list produced by the lexer
- *	doc - a variable to receive the resulting abstract syntax tree (AST)
- * OPERATION:
- *	The token list is emptied during parsing, so far as the parsing
- *	succeeds. This allows the caller to inspect the line/char position
- *	of the next token (where the error likely is) and report that hint to
- *	the user in the form of an error message.
- * RETURN:
- *	GRAPHQL_SUCCESS or an error string.
- */
-const char *graphql_parse(struct list_head *tokens, struct graphql_executable_document **doc);
-
-/* The lexer and parser in one function, for convenience. */
-const char *graphql_lexparse(const tal_t *ctx, const char *input, struct list_head **tokens, struct graphql_executable_document **doc);
-
-#endif
-

ccan/ccan/htable/htable.c

@@ -86,16 +86,14 @@ void htable_init(struct htable *ht,
 	ht->table = &ht->common_bits;
 }
 
-/* Fill to 87.5% */
 static inline size_t ht_max(const struct htable *ht)
 {
-	return ((size_t)7 << ht->bits) / 8;
+	return ((size_t)3 << ht->bits) / 4;
 }
 
-/* Clean deleted if we're full, and more than 12.5% deleted */
-static inline size_t ht_max_deleted(const struct htable *ht)
+static inline size_t ht_max_with_deleted(const struct htable *ht)
 {
-	return ((size_t)1 << ht->bits) / 8;
+	return ((size_t)9 << ht->bits) / 10;
 }
 
 bool htable_init_sized(struct htable *ht,
@@ -105,7 +103,7 @@ bool htable_init_sized(struct htable *ht,
 	htable_init(ht, rehash, priv);
 
 	/* Don't go insane with sizing. */
-	for (ht->bits = 1; ht_max(ht) < expect; ht->bits++) {
+	for (ht->bits = 1; ((size_t)3 << ht->bits) / 4 < expect; ht->bits++) {
 		if (ht->bits == 30)
 			break;
 	}
@@ -197,83 +195,12 @@ void *htable_prev_(const struct htable *ht, struct htable_iter *i)
 	for (;;) {
 		if (!i->off)
 			return NULL;
-		i->off--;
+		i->off --;
 		if (entry_is_valid(ht->table[i->off]))
 			return get_raw_ptr(ht, ht->table[i->off]);
 	}
 }
 
-/* Another bit currently in mask needs to be exposed, so that a bucket with p in
- * it won't appear invalid */
-static COLD void unset_another_common_bit(struct htable *ht,
-					  uintptr_t *maskdiff,
-					  const void *p)
-{
-	size_t i;
-
-	for (i = sizeof(uintptr_t) * CHAR_BIT - 1; i > 0; i--) {
-		if (((uintptr_t)p & ((uintptr_t)1 << i))
-		    && ht->common_mask & ~*maskdiff & ((uintptr_t)1 << i))
-			break;
-	}
-	/* There must have been one, right? */
-	assert(i > 0);
-
-	*maskdiff |= ((uintptr_t)1 << i);
-}
-
-/* We want to change the common mask: this fixes up the table */
-static COLD void fixup_table_common(struct htable *ht, uintptr_t maskdiff)
-{
-	size_t i;
-	uintptr_t bitsdiff;
-
-again:
-	bitsdiff = ht->common_bits & maskdiff;
-
-	for (i = 0; i < (size_t)1 << ht->bits; i++) {
-		uintptr_t e;
-		if (!entry_is_valid(e = ht->table[i]))
-			continue;
-
-		/* Clear the bits no longer in the mask, set them as
-		 * expected. */
-		e &= ~maskdiff;
-		e |= bitsdiff;
-		/* If this made it invalid, restart with more exposed */
-		if (!entry_is_valid(e)) {
-			unset_another_common_bit(ht, &maskdiff, get_raw_ptr(ht, e));
-			goto again;
-		}
-		ht->table[i] = e;
-	}
-
-	/* Take away those bits from our mask, bits and perfect bit. */
-	ht->common_mask &= ~maskdiff;
-	ht->common_bits &= ~maskdiff;
-	if (ht_perfect_mask(ht) & maskdiff)
-		ht->perfect_bitnum = NO_PERFECT_BIT;
-}
-
-/* Limited recursion */
-static void ht_add(struct htable *ht, const void *new, size_t h);
-
-/* We tried to add this entry, but it looked invalid!  We need to
- * let another pointer bit through mask */
-static COLD void update_common_fix_invalid(struct htable *ht, const void *p, size_t h)
-{
-	uintptr_t maskdiff;
-
-	assert(ht->elems != 0);
-
-	maskdiff = 0;
-	unset_another_common_bit(ht, &maskdiff, p);
-	fixup_table_common(ht, maskdiff);
-
-	/* Now won't recurse */
-	ht_add(ht, p, h);
-}
-
 /* This does not expand the hash table, that's up to caller. */
 static void ht_add(struct htable *ht, const void *new, size_t h)
 {
@@ -287,8 +214,6 @@ static void ht_add(struct htable *ht, const void *new, size_t h)
 		i = (i + 1) & ((1 << ht->bits)-1);
 	}
 	ht->table[i] = make_hval(ht, new, get_hash_ptr_bits(ht, h)|perfect);
-	if (!entry_is_valid(ht->table[i]))
-		update_common_fix_invalid(ht, new, h);
 }
 
 static COLD bool double_table(struct htable *ht)
@@ -358,10 +283,20 @@ static COLD void rehash_table(struct htable *ht)
 /* We stole some bits, now we need to put them back... */
 static COLD void update_common(struct htable *ht, const void *p)
 {
-	uintptr_t maskdiff;
+	unsigned int i;
+	uintptr_t maskdiff, bitsdiff;
 
 	if (ht->elems == 0) {
-		ht->common_mask = -1;
+		/* Always reveal one bit of the pointer in the bucket,
+		 * so it's not zero or HTABLE_DELETED (1), even if
+		 * hash happens to be 0.  Assumes (void *)1 is not a
+		 * valid pointer. */
+		for (i = sizeof(uintptr_t)*CHAR_BIT - 1; i > 0; i--) {
+			if ((uintptr_t)p & ((uintptr_t)1 << i))
+				break;
+		}
+
+		ht->common_mask = ~((uintptr_t)1 << i);
 		ht->common_bits = ((uintptr_t)p & ht->common_mask);
 		ht->perfect_bitnum = 0;
 		(void)htable_debug(ht, HTABLE_LOC);
@@ -371,25 +306,33 @@ static COLD void update_common(struct htable *ht, const void *p)
 	/* Find bits which are unequal to old common set. */
 	maskdiff = ht->common_bits ^ ((uintptr_t)p & ht->common_mask);
 
-	fixup_table_common(ht, maskdiff);
+	/* These are the bits which go there in existing entries. */
+	bitsdiff = ht->common_bits & maskdiff;
+
+	for (i = 0; i < (size_t)1 << ht->bits; i++) {
+		if (!entry_is_valid(ht->table[i]))
+			continue;
+		/* Clear the bits no longer in the mask, set them as
+		 * expected. */
+		ht->table[i] &= ~maskdiff;
+		ht->table[i] |= bitsdiff;
+	}
+
+	/* Take away those bits from our mask, bits and perfect bit. */
+	ht->common_mask &= ~maskdiff;
+	ht->common_bits &= ~maskdiff;
+	if (ht_perfect_mask(ht) & maskdiff)
+		ht->perfect_bitnum = NO_PERFECT_BIT;
 	(void)htable_debug(ht, HTABLE_LOC);
 }
 
 bool htable_add_(struct htable *ht, size_t hash, const void *p)
 {
-	/* Cannot insert NULL, or (void *)1. */
+	if (ht->elems+1 > ht_max(ht) && !double_table(ht))
+		return false;
+	if (ht->elems+1 + ht->deleted > ht_max_with_deleted(ht))
+		rehash_table(ht);
 	assert(p);
-	assert(entry_is_valid((uintptr_t)p));
-
-	/* Getting too full? */
-	if (ht->elems+1 + ht->deleted > ht_max(ht)) {
-		/* If we're more than 1/8 deleted, clean those,
-		 * otherwise double table size. */
-		if (ht->deleted > ht_max_deleted(ht))
-			rehash_table(ht);
-		else if (!double_table(ht))
-			return false;
-	}
 	if (((uintptr_t)p & ht->common_mask) != ht->common_bits)
 		update_common(ht, p);
 
@@ -418,12 +361,8 @@ void htable_delval_(struct htable *ht, struct htable_iter *i)
 	assert(entry_is_valid(ht->table[i->off]));
 
 	ht->elems--;
-	/* Cheap test: if the next bucket is empty, don't need delete marker */
-	if (ht->table[hash_bucket(ht, i->off+1)] != 0) {
-		ht->table[i->off] = HTABLE_DELETED;
-		ht->deleted++;
-	} else
-		ht->table[i->off] = 0;
+	ht->table[i->off] = HTABLE_DELETED;
+	ht->deleted++;
 }
 
 void *htable_pick_(const struct htable *ht, size_t seed, struct htable_iter *i)

ccan/ccan/htable/htable.h

@@ -132,7 +132,7 @@ bool htable_copy_(struct htable *dst, const struct htable *src);
  * htable_add - add a pointer into a hash table.
  * @ht: the htable
  * @hash: the hash value of the object
- * @p: the non-NULL pointer (also cannot be (void *)1).
+ * @p: the non-NULL pointer
  *
  * Also note that this can only fail due to allocation failure.  Otherwise, it
  * returns true.

ccan/ccan/htable/htable_type.h

@@ -1,23 +1,19 @@
 /* Licensed under LGPLv2+ - see LICENSE file for details */
 #ifndef CCAN_HTABLE_TYPE_H
 #define CCAN_HTABLE_TYPE_H
-#include "config.h"
-#include <assert.h>
 #include <ccan/htable/htable.h>
 #include <ccan/compiler/compiler.h>
+#include "config.h"
 
 /**
- * HTABLE_DEFINE_NODUPS_TYPE/HTABLE_DEFINE_DUPS_TYPE - create a set of htable ops for a type
+ * HTABLE_DEFINE_TYPE - create a set of htable ops for a type
  * @type: a type whose pointers will be values in the hash.
  * @keyof: a function/macro to extract a key: <keytype> @keyof(const type *elem)
  * @hashfn: a hash function for a @key: size_t @hashfn(const <keytype> *)
  * @eqfn: an equality function keys: bool @eqfn(const type *, const <keytype> *)
  * @prefix: a prefix for all the functions to define (of form <name>_*)
  *
- * There are two variants, one of which allows duplicate keys, and one which
- * does not.  The defined functions differ in some cases, as shown below.
- *
- * NULL values may not be placed into the hash table (nor (void *)1).
+ * NULL values may not be placed into the hash table.
  *
  * This defines the type hashtable type and an iterator type:
  *	struct <name>;
@@ -37,18 +33,15 @@
  *
  * Delete and delete-by key return true if it was in the set:
  *	bool <name>_del(struct <name> *ht, const <type> *e);
- *	bool <name>_delkey(struct <name> *ht, const <keytype> *k) (NODUPS only);
+ *	bool <name>_delkey(struct <name> *ht, const <keytype> *k);
  *
  * Delete by iterator:
  *	bool <name>_delval(struct <name> *ht, struct <name>_iter *i);
  *
- * Find and return the matching element, or NULL:
- *	type *<name>_get(const struct @name *ht, const <keytype> *k) (NODUPS only);
+ * Find and return the (first) matching element, or NULL:
+ *	type *<name>_get(const struct @name *ht, const <keytype> *k);
  *
- * Test for an element:
- *	bool <name>_exists(const struct @name *ht, const <keytype> *k);
- *
- * Find and return all matching elements, or NULL (DUPS only):
+ * Find and return all matching elements, or NULL:
  *	type *<name>_getfirst(const struct @name *ht, const <keytype> *k,
  *			      struct <name>_iter *i);
  *	type *<name>_getnext(const struct @name *ht, const <keytype> *k,
@@ -66,7 +59,7 @@
  * You can use HTABLE_INITIALIZER like so:
  *	struct <name> ht = { HTABLE_INITIALIZER(ht.raw, <name>_hash, NULL) };
  */
-#define HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
+#define HTABLE_DEFINE_TYPE(type, keyof, hashfn, eqfn, name)		\
 	struct name { struct htable raw; };				\
 	struct name##_iter { struct htable_iter i; };			\
 	static inline size_t name##_hash(const void *elem, void *priv)	\
@@ -96,33 +89,66 @@
 	{								\
 		return htable_copy(&dst->raw, &src->raw);		\
 	}								\
+	static inline bool name##_add(struct name *ht, const type *elem) \
+	{								\
+		return htable_add(&ht->raw, hashfn(keyof(elem)), elem);	\
+	}								\
 	static inline UNNEEDED bool name##_del(struct name *ht,		\
 					       const type *elem)	\
 	{								\
 		return htable_del(&ht->raw, hashfn(keyof(elem)), elem);	\
 	}								\
-	static inline UNNEEDED type *name##_getmatch_(const struct name *ht, \
-				         const HTABLE_KTYPE(keyof, type) k, \
-				         size_t h,			\
-				         type *v,			\
-					 struct htable_iter *iter)	\
-	{								\
-		while (v) {						\
-			if (eqfn(v, k))					\
-				break;					\
-			v = htable_nextval(&ht->raw, iter, h);		\
-		}							\
-		return v;						\
-	}								\
-	static inline UNNEEDED bool name##_exists(const struct name *ht,	\
+	static inline UNNEEDED type *name##_get(const struct name *ht,	\
 				       const HTABLE_KTYPE(keyof, type) k) \
 	{								\
 		struct htable_iter i;					\
 		size_t h = hashfn(k);					\
-		void *v;						\
+		void *c;						\
 									\
-		v = htable_firstval(&ht->raw, &i, h);			\
-		return name##_getmatch_(ht, k, h, v, &i) != NULL;	\
+		for (c = htable_firstval(&ht->raw,&i,h);		\
+		     c;							\
+		     c = htable_nextval(&ht->raw,&i,h)) {		\
+			if (eqfn(c, k))					\
+				return c;				\
+		}							\
+		return NULL;						\
+	}								\
+	static inline UNNEEDED type *name##_getmatch_(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+				         size_t h,			\
+				         type *v,			\
+					 struct name##_iter *iter)	\
+	{								\
+		while (v) {						\
+			if (eqfn(v, k))					\
+				break;					\
+			v = htable_nextval(&ht->raw, &iter->i, h);	\
+		}							\
+		return v;						\
+	}								\
+	static inline UNNEEDED type *name##_getfirst(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+					 struct name##_iter *iter)	\
+	{								\
+		size_t h = hashfn(k);					\
+		type *v = htable_firstval(&ht->raw, &iter->i, h);	\
+		return name##_getmatch_(ht, k, h, v, iter);			\
+	}								\
+	static inline UNNEEDED type *name##_getnext(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+					 struct name##_iter *iter)	\
+	{								\
+		size_t h = hashfn(k);					\
+		type *v = htable_nextval(&ht->raw, &iter->i, h);	\
+		return name##_getmatch_(ht, k, h, v, iter);		\
+	}								\
+	static inline UNNEEDED bool name##_delkey(struct name *ht,	\
+					 const HTABLE_KTYPE(keyof, type) k) \
+	{								\
+		type *elem = name##_get(ht, k);				\
+		if (elem)						\
+			return name##_del(ht, elem);			\
+		return false;						\
 	}								\
 	static inline UNNEEDED void name##_delval(struct name *ht,	\
 						  struct name##_iter *iter) \
@@ -133,7 +159,8 @@
 						size_t seed,		\
 						struct name##_iter *iter) \
 	{								\
-		return htable_pick(&ht->raw, seed, iter ? &iter->i : NULL); \
+		/* Note &iter->i == NULL iff iter is NULL */		\
+		return htable_pick(&ht->raw, seed, &iter->i);			\
 	}								\
 	static inline UNNEEDED type *name##_first(const struct name *ht, \
 					 struct name##_iter *iter)	\
@@ -151,64 +178,6 @@
 		return htable_prev(&ht->raw, &iter->i);			\
 	}
 
-#define HTABLE_DEFINE_NODUPS_TYPE(type, keyof, hashfn, eqfn, name)	\
-	HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
-	static inline UNNEEDED type *name##_get(const struct name *ht,	\
-				       const HTABLE_KTYPE(keyof, type) k) \
-	{								\
-		struct htable_iter i;					\
-		size_t h = hashfn(k);					\
-		void *v;						\
-									\
-		v = htable_firstval(&ht->raw, &i, h);			\
-		return name##_getmatch_(ht, k, h, v, &i);			\
-	}								\
-	static inline bool name##_add(struct name *ht, const type *elem) \
-	{								\
-		/* Open-coded for slightly more efficiency */		\
-		const HTABLE_KTYPE(keyof, type) k = keyof(elem);	\
-		struct htable_iter i;					\
-		size_t h = hashfn(k);					\
-		void *v;						\
-									\
-		v = htable_firstval(&ht->raw, &i, h);			\
-		assert(!name##_getmatch_(ht, k, h, v, &i));		\
-		return htable_add(&ht->raw, h, elem);			\
-	}								\
-	static inline UNNEEDED bool name##_delkey(struct name *ht,	\
-					 const HTABLE_KTYPE(keyof, type) k) \
-	{								\
-		type *elem = name##_get(ht, k);				\
-		if (elem)						\
-			return name##_del(ht, elem);			\
-		return false;						\
-	}
-
-#define HTABLE_DEFINE_DUPS_TYPE(type, keyof, hashfn, eqfn, name)	\
-	HTABLE_DEFINE_TYPE_CORE(type, keyof, hashfn, eqfn, name)	\
-	static inline bool name##_add(struct name *ht, const type *elem) \
-	{								\
-		const HTABLE_KTYPE(keyof, type) k = keyof(elem);	\
-		return htable_add(&ht->raw, hashfn(k), elem);		\
-	}								\
-	static inline UNNEEDED type *name##_getfirst(const struct name *ht, \
-				         const HTABLE_KTYPE(keyof, type) k, \
-					 struct name##_iter *iter)	\
-	{								\
-		size_t h = hashfn(k);					\
-		type *v = htable_firstval(&ht->raw, &iter->i, h);	\
-		return name##_getmatch_(ht, k, h, v, &iter->i);		\
-	}								\
-	static inline UNNEEDED type *name##_getnext(const struct name *ht, \
-				         const HTABLE_KTYPE(keyof, type) k, \
-					 struct name##_iter *iter)	\
-	{								\
-		size_t h = hashfn(k);					\
-		type *v = htable_nextval(&ht->raw, &iter->i, h);	\
-		return name##_getmatch_(ht, k, h, v, &iter->i);		\
-	}
-
-
 #if HAVE_TYPEOF
 #define HTABLE_KTYPE(keyof, type) typeof(keyof((const type *)NULL))
 #else

ccan/ccan/htable/test/run-clash.c

@@ -1,31 +0,0 @@
-#include <ccan/htable/htable.h>
-#include <ccan/htable/htable.c>
-#include <ccan/tap/tap.h>
-#include <stdbool.h>
-#include <string.h>
-
-/* Clashy hash */
-static size_t hash(const void *elem, void *unused UNNEEDED)
-{
-	return 0;
-}
-
-int main(void)
-{
-	struct htable ht;
-
-	plan_tests(254 * 253);
-	/* We try to get two elements which clash */
-	for (size_t i = 2; i < 256; i++) {
-		for (size_t j = 2; j < 256; j++) {
-			if (i == j)
-				continue;
-			htable_init(&ht, hash, NULL);
-			htable_add(&ht, hash((void *)i, NULL), (void *)i);
-			htable_add(&ht, hash((void *)j, NULL), (void *)j);
-			ok1(htable_check(&ht, "test"));
-			htable_clear(&ht);
-		}
-	}
-	return exit_status();
-}

ccan/ccan/htable/test/run-debug.c

@@ -107,7 +107,7 @@ static bool check_mask(struct htable *ht, uint64_t val[], unsigned num)
 
 int main(void)
 {
-	unsigned int i;
+	unsigned int i, weight;
 	uintptr_t perfect_bit;
 	struct htable ht;
 	uint64_t val[NUM_VALS];
@@ -131,7 +131,14 @@ int main(void)
 	add_vals(&ht, val, 0, 1);
 	ok1(ht.bits == 1);
 	ok1(ht_max(&ht) == 1);
-	ok1(ht.common_mask == -1);
+	weight = 0;
+	for (i = 0; i < sizeof(ht.common_mask) * CHAR_BIT; i++) {
+		if (ht.common_mask & ((uintptr_t)1 << i)) {
+			weight++;
+		}
+	}
+	/* Only one bit should be clear. */
+	ok1(weight == i-1);
 
 	/* Mask should be set. */
 	ok1(check_mask(&ht, val, 1));

ccan/ccan/htable/test/run-type-int.c

@@ -38,7 +38,7 @@ static bool cmp(const struct obj *obj, const unsigned int key)
 	return obj->key == key;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
+HTABLE_DEFINE_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
 
 static void add_vals(struct htable_obj *ht,
 		     struct obj val[], unsigned int num)
@@ -112,19 +112,14 @@ static bool check_mask(struct htable *ht, const struct obj val[], unsigned num)
 	return true;
 }
 
-/* This variant allows duplicates! */
-HTABLE_DEFINE_DUPS_TYPE(struct obj, objkey, objhash, cmp, htable_obj_dups);
-
 int main(void)
 {
 	unsigned int i;
 	struct htable_obj ht, ht2;
-	struct htable_obj_dups ht_dups;
 	struct obj val[NUM_VALS], *result;
 	unsigned int dne;
 	void *p;
 	struct htable_obj_iter iter;
-	struct htable_obj_dups_iter dups_iter;
 
 	plan_tests(29);
 	for (i = 0; i < NUM_VALS; i++)
@@ -188,35 +183,32 @@ int main(void)
 	del_vals_bykey(&ht, val, NUM_VALS);
 	del_vals_bykey(&ht2, val, NUM_VALS);
 
-	/* Duplicates-allowed tests */
-	htable_obj_dups_init(&ht_dups);
 	/* Write two of the same value. */
 	val[1] = val[0];
-	htable_obj_dups_add(&ht_dups, &val[0]);
-	htable_obj_dups_add(&ht_dups, &val[1]);
+	htable_obj_add(&ht, &val[0]);
+	htable_obj_add(&ht, &val[1]);
 	i = 0;
 
-	result = htable_obj_dups_getfirst(&ht_dups, i, &dups_iter);
+	result = htable_obj_getfirst(&ht, i, &iter);
 	ok1(result == &val[0] || result == &val[1]);
 	if (result == &val[0]) {
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[0]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[0]));
+		ok1(htable_obj_getfirst(&ht, i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 	} else {
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[1]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[1]));
+		ok1(htable_obj_getfirst(&ht, i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, i, &iter) == NULL);
 	}
 
-	htable_obj_dups_clear(&ht_dups);
 	htable_obj_clear(&ht);
 	htable_obj_clear(&ht2);
 	return exit_status();

ccan/ccan/htable/test/run-type.c

@@ -33,10 +33,7 @@ static bool cmp(const struct obj *obj, const unsigned int *key)
 	return obj->key == *key;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(struct obj, objkey, objhash, cmp,
-			  htable_obj);
-HTABLE_DEFINE_DUPS_TYPE(struct obj, objkey, objhash, cmp,
-			htable_obj_dups);
+HTABLE_DEFINE_TYPE(struct obj, objkey, objhash, cmp, htable_obj);
 
 static void add_vals(struct htable_obj *ht,
 		     struct obj val[], unsigned int num)
@@ -114,14 +111,12 @@ int main(void)
 {
 	unsigned int i;
 	struct htable_obj ht, ht2;
-	struct htable_obj_dups ht_dups;
 	struct obj val[NUM_VALS], *result;
 	unsigned int dne;
 	void *p;
 	struct htable_obj_iter iter;
-	struct htable_obj_dups_iter dups_iter;
 
-	plan_tests(36);
+	plan_tests(35);
 	for (i = 0; i < NUM_VALS; i++)
 		val[i].key = i;
 	dne = i;
@@ -187,37 +182,32 @@ int main(void)
 	del_vals_bykey(&ht, val, NUM_VALS);
 	del_vals_bykey(&ht2, val, NUM_VALS);
 
-	/* Duplicates-allowed tests */
-	htable_obj_dups_init(&ht_dups);
-
 	/* Write two of the same value. */
 	val[1] = val[0];
-	htable_obj_dups_add(&ht_dups, &val[0]);
-	htable_obj_dups_add(&ht_dups, &val[1]);
+	htable_obj_add(&ht, &val[0]);
+	htable_obj_add(&ht, &val[1]);
 	i = 0;
 
-	result = htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter);
+	result = htable_obj_getfirst(&ht, &i, &iter);
 	ok1(result == &val[0] || result == &val[1]);
 	if (result == &val[0]) {
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[0]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter) == &val[1]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[0]));
+		ok1(htable_obj_getfirst(&ht, &i, &iter) == &val[1]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 	} else {
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 
 		/* Deleting first should make us iterate over the other. */
-		ok1(htable_obj_dups_del(&ht_dups, &val[1]));
-		ok1(htable_obj_dups_getfirst(&ht_dups, &i, &dups_iter) == &val[0]);
-		ok1(htable_obj_dups_getnext(&ht_dups, &i, &dups_iter) == NULL);
+		ok1(htable_obj_del(&ht, &val[1]));
+		ok1(htable_obj_getfirst(&ht, &i, &iter) == &val[0]);
+		ok1(htable_obj_getnext(&ht, &i, &iter) == NULL);
 	}
 
-	htable_obj_dups_clear(&ht_dups);
-	ok1(htable_obj_dups_count(&ht_dups) == 0);
 	htable_obj_clear(&ht);
 	ok1(htable_obj_count(&ht) == 0);
 	htable_obj_clear(&ht2);

ccan/ccan/htable/test/run.c

@@ -97,7 +97,7 @@ static bool check_mask(struct htable *ht, uint64_t val[], unsigned num)
 
 int main(void)
 {
-	unsigned int i;
+	unsigned int i, weight;
 	uintptr_t perfect_bit;
 	struct htable ht;
 	uint64_t val[NUM_VALS];
@@ -122,7 +122,14 @@ int main(void)
 	add_vals(&ht, val, 0, 1);
 	ok1(ht.bits == 1);
 	ok1(ht_max(&ht) == 1);
-	ok1(ht.common_mask == -1);
+	weight = 0;
+	for (i = 0; i < sizeof(ht.common_mask) * CHAR_BIT; i++) {
+		if (ht.common_mask & ((uintptr_t)1 << i)) {
+			weight++;
+		}
+	}
+	/* Only one bit should be clear. */
+	ok1(weight == i-1);
 
 	/* Mask should be set. */
 	ok1(check_mask(&ht, val, 1));

ccan/ccan/htable/tools/Makefile

@@ -4,10 +4,9 @@ CFLAGS=-Wall -Werror -O3 -I$(CCANDIR)
 
 CCAN_OBJS:=ccan-tal.o ccan-tal-str.o ccan-tal-grab_file.o ccan-take.o ccan-time.o ccan-str.o ccan-noerr.o ccan-list.o
 
-all: speed stringspeed hsearchspeed density
+all: speed stringspeed hsearchspeed
 
 speed: speed.o hash.o $(CCAN_OBJS)
-density: density.o hash.o $(CCAN_OBJS)
 
 speed.o: speed.c ../htable.h ../htable.c
 

ccan/ccan/htable/tools/density.c

@@ -1,107 +0,0 @@
-/* Density measurements for hashtables. */
-#include <ccan/err/err.h>
-#include <ccan/htable/htable_type.h>
-#include <ccan/htable/htable.c>
-#include <ccan/hash/hash.h>
-#include <ccan/ptrint/ptrint.h>
-#include <ccan/time/time.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-/* We don't actually hash objects: we put values in as if they were ptrs */
-static uintptr_t key(const ptrint_t *obj)
-{
-	return ptr2int(obj);
-}
-
-static size_t hash_uintptr(uintptr_t key)
-{
-	return hashl(&key, 1, 0);
-}
-
-static bool cmp(const ptrint_t *p, uintptr_t k)
-{
-	return key(p) == k;
-}
-
-HTABLE_DEFINE_NODUPS_TYPE(ptrint_t, key, hash_uintptr, cmp, htable_ptrint);
-
-/* Nanoseconds per operation */
-static size_t normalize(const struct timeabs *start,
-			const struct timeabs *stop,
-			unsigned int num)
-{
-	return time_to_nsec(time_divide(time_between(*stop, *start), num));
-}
-
-static size_t average_run(const struct htable_ptrint *ht, size_t count, size_t *longest)
-{
-	size_t i, total = 0;
-
-	*longest = 0;
-	for (i = 0; i < count; i++) {
-		size_t h = hash_uintptr(i + 2);
-		size_t run = 1;
-
-		while (get_raw_ptr(&ht->raw, ht->raw.table[h % ((size_t)1 << ht->raw.bits)]) != int2ptr(i + 2)) {
-			h++;
-			run++;
-		}
-		total += run;
-		if (run > *longest)
-			*longest = run;
-	}
-	return total / count;
-}
-
-int main(int argc, char *argv[])
-{
-	unsigned int i;
-	size_t num;
-	struct timeabs start, stop;
-	struct htable_ptrint ht;
-
-	if (argc != 2)
-		errx(1, "Usage: density <power-of-2-tablesize>");
-
-	num = atoi(argv[1]);
-
-	printf("Total buckets, buckets used, nanoseconds search time per element, avg run, longest run\n");
-	for (i = 1; i <= 99; i++) {
-		uintptr_t j;
-		struct htable_ptrint_iter it;
-		size_t count, avg_run, longest_run;
-		ptrint_t *p;
-
-		htable_ptrint_init_sized(&ht, num * 3 / 4);
-		assert((1 << ht.raw.bits) == num);
-
-		/* Can't put 0 or 1 in the hash table: multiply by a prime. */
-		for (j = 0; j < num * i / 100; j++) {
-			htable_ptrint_add(&ht, int2ptr(j + 2));
-			/* stop it from doubling! */
-			ht.raw.elems = num / 2;
-		}
-		/* Must not have changed! */
-		assert((1 << ht.raw.bits) == num);
-
-		/* Clean cache */
-		count = 0;
-		for (p = htable_ptrint_first(&ht, &it); p; p = htable_ptrint_next(&ht, &it))
-			count++;
-		assert(count == num * i / 100);
-		start = time_now();
-		for (j = 0; j < count; j++)
-			assert(htable_ptrint_get(&ht, j + 2));
-		stop = time_now();
-		avg_run = average_run(&ht, count, &longest_run);
-		printf("%zu,%zu,%zu,%zu,%zu\n",
-		       num, count, normalize(&start, &stop, count), avg_run, longest_run);
-		fflush(stdout);
-		htable_ptrint_clear(&ht);
-	}
-
-	return 0;
-}

ccan/ccan/htable/tools/speed.c

@@ -33,7 +33,7 @@ static bool cmp(const struct object *object, const unsigned int *key)
 	return object->key == *key;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(struct object, objkey, hash_obj, cmp, htable_obj);
+HTABLE_DEFINE_TYPE(struct object, objkey, hash_obj, cmp, htable_obj);
 
 static unsigned int popcount(unsigned long val)
 {

ccan/ccan/htable/tools/stringspeed.c

@@ -31,7 +31,7 @@ static bool cmp(const char *obj, const char *key)
 	return strcmp(obj, key) == 0;
 }
 
-HTABLE_DEFINE_NODUPS_TYPE(char, strkey, hash_str, cmp, htable_str);
+HTABLE_DEFINE_TYPE(char, strkey, hash_str, cmp, htable_str);
 
 /* Nanoseconds per operation */
 static size_t normalize(const struct timeabs *start,

ccan/ccan/ilog/ilog.h

@@ -120,10 +120,7 @@ int ilog64_nz(uint64_t _v) CONST_FUNCTION;
 #endif
 
 #ifdef builtin_ilog32_nz
-/* This used to be builtin_ilog32_nz(_v)&-!!(_v), which means it zeroes out
- * the undefined builtin_ilog32_nz(0) return.  But clang UndefinedBehaviorSantizer
- * complains, so do the branch: */
-#define ilog32(_v) ((_v) ? builtin_ilog32_nz(_v) : 0)
+#define ilog32(_v) (builtin_ilog32_nz(_v)&-!!(_v))
 #define ilog32_nz(_v) builtin_ilog32_nz(_v)
 #else
 #define ilog32_nz(_v) ilog32(_v)
@@ -131,7 +128,7 @@ int ilog64_nz(uint64_t _v) CONST_FUNCTION;
 #endif /* builtin_ilog32_nz */
 
 #ifdef builtin_ilog64_nz
-#define ilog32(_v) ((_v) ? builtin_ilog32_nz(_v) : 0)
+#define ilog64(_v) (builtin_ilog64_nz(_v)&-!!(_v))
 #define ilog64_nz(_v) builtin_ilog64_nz(_v)
 #else
 #define ilog64_nz(_v) ilog64(_v)

ccan/ccan/intmap/benchmark/speed.c

@@ -59,7 +59,7 @@ static bool eqfn(const struct htable_elem *elem, const uint64_t index)
 {
 	return elem->index == index;
 }
-HTABLE_DEFINE_NODUPS_TYPE(struct htable_elem, keyof, hashfn, eqfn, hash);
+HTABLE_DEFINE_TYPE(struct htable_elem, keyof, hashfn, eqfn, hash);
 
 static bool check_val(intmap_index_t i, uint64_t *v, uint64_t *expected)
 {

ccan/ccan/io/benchmarks/Makefile

@@ -1,17 +1,16 @@
-ALL:=run-loop run-different-speed run-length-prefix run-stream-many
+ALL:=run-loop run-different-speed run-length-prefix
 CCANDIR:=../../..
 CFLAGS:=-Wall -I$(CCANDIR) -O3 -flto
 LDFLAGS:=-O3 -flto
 LDLIBS:=-lrt
 
-OBJS:=time.o poll.o io.o err.o timer.o list.o ccan-tal.o ccan-take.o ccan-ilog.o
+OBJS:=time.o poll.o io.o err.o timer.o list.o
 
 default: $(ALL)
 
 run-loop: run-loop.o $(OBJS)
 run-different-speed: run-different-speed.o $(OBJS)
 run-length-prefix: run-length-prefix.o $(OBJS)
-run-stream-many: run-stream-many.o $(OBJS)
 
 time.o: $(CCANDIR)/ccan/time/time.c
 	$(CC) $(CFLAGS) -c -o $@ $<
@@ -25,12 +24,6 @@ io.o: $(CCANDIR)/ccan/io/io.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 err.o: $(CCANDIR)/ccan/err/err.c
 	$(CC) $(CFLAGS) -c -o $@ $<
-ccan-ilog.o: $(CCANDIR)/ccan/ilog/ilog.c
-	$(CC) $(CFLAGS) -c -o $@ $<
-ccan-tal.o: $(CCANDIR)/ccan/tal/tal.c
-	$(CC) $(CFLAGS) -c -o $@ $<
-ccan-take.o: $(CCANDIR)/ccan/take/take.c
-	$(CC) $(CFLAGS) -c -o $@ $<
 
 clean:
 	rm -f *.o $(ALL)

ccan/ccan/io/benchmarks/run-stream-many.c

@@ -1,132 +0,0 @@
-/* Wait for many fds to connect, then try to stream the file to some of them in small chunks.
- *
- * This approximates the connectd behaviour in CLN, where we send gossip to peers.
- */
-#include <ccan/io/io.h>
-#include <ccan/ptrint/ptrint.h>
-#include <ccan/time/time.h>
-#include <inttypes.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-#include <err.h>
-#include <signal.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-
-/* We expect num_expected connections, and how many will be writers */
-static size_t max_readers, max_writers;
-
-/* How many raeaders and writers still going */
-static size_t num_readers, num_writers;
-
-/* How many times to do the write */
-static size_t write_iterations;
-
-/* The buffer to write */
-static char writebuf[256];
-
-/* We need this for readers, though we don't actually care! */
-static size_t len_ignored;
-
-struct timemono start_time;
-
-static void finished(void)
-{
-	struct timerel elapsed = timemono_since(start_time);
-	printf("Finished: %"PRIu64"usec\n", time_to_usec(elapsed));
-	exit(0);
-}
-
-static struct io_plan *write_loop(struct io_conn *conn, ptrint_t *iter)
-{
-	ptrdiff_t n = ptr2int(iter);
-
-	if (n > write_iterations) {
-		--num_writers;
-		if (num_writers == 0)
-			finished();
-		return io_wait(conn, conn, io_never, NULL);
-	}
-	return io_write(conn, writebuf, sizeof(writebuf), write_loop, int2ptr(n + 1));
-}
-
-static struct io_plan *read_loop(struct io_conn *conn, void *unused)
-{
-	return io_read_partial(conn, writebuf, sizeof(writebuf), &len_ignored, read_loop, unused);
-}
-
-static void reader_failed(struct io_conn *conn, intptr_t *num)
-{
-	err(1, "Reader %zu/%zu", (size_t)ptr2int(num), max_readers);
-}
-
-static void writer_failed(struct io_conn *conn, intptr_t *num)
-{
-	err(1, "Writer %zu/%zu", (size_t)ptr2int(num), max_writers);
-}
-
-static struct io_plan *connection_in(struct io_conn *conn, void *sleep_on)
-{
-	if (num_readers < max_readers) {
-		printf("r");
-		fflush(stdout);
-		num_readers++;
-		io_set_finish(conn, reader_failed, int2ptr(num_readers));
-		return read_loop(conn, NULL);
-	}
-
-	/* We assign writers last: not sure it matters, but it's more reflective
-	 * of lightning where more recent connections tend to ask for gossip */
-	num_writers++;
-	printf("w");
-	fflush(stdout);
-
-	io_set_finish(conn, writer_failed, int2ptr(num_writers));
-	io_set_finish(conn, writer_failed, NULL);
-	if (num_writers < max_writers)
-		return io_wait(conn, sleep_on, write_loop, int2ptr(0));
-
-	/* Everyone is connected.  Wake them and start final one */
-	io_wake(sleep_on);
-	printf("Starting!\n");
-	start_time = time_mono();
-	return write_loop(conn, int2ptr(0));
-}
-
-int main(int argc, char *argv[])
-{
-	int fd;
-	struct sockaddr_in s4;
-	int on = 1;
-
-	if (argc != 5)
-		errx(1, "Usage: <portnum> <num-idle> <num-streaming> <mb-streamed>");
-
-	memset(&s4, 0, sizeof(s4));
-	s4.sin_family = AF_INET;
-	s4.sin_port = htons(atol(argv[1]));
-	s4.sin_addr.s_addr = INADDR_ANY;
-
-	max_readers = atol(argv[2]);
-	max_writers = atol(argv[3]);
-	write_iterations = atol(argv[4]) * (1024 * 1024 / sizeof(writebuf));
-
-	fd = socket(AF_INET, SOCK_STREAM, 0);
-	if (fd < 0)
-		err(1, "Creating socket");
-
-	/* Re-use, please.. */
-	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)))
-		err(1, "Setting reuseaddr");
-
-	if (bind(fd, &s4, sizeof(s4)) != 0)
-		err(1, "Binding");
-
-	if (listen(fd, 1) != 0)
-		err(1, "Listening");
-
-	io_new_listener(NULL, fd, connection_in, &s4);
-	io_loop(NULL, NULL);
-	errx(1, "Sockets exited?");
-}

ccan/ccan/io/fdpass/_info

@@ -32,20 +32,12 @@
  *					read_more, buf);
  *	}
  *
- *	// Clean up allocation so -fsanitize=address doesn't see leak!
- *	static void free_buf(struct io_conn *c, struct buf *buf)
- *	{
- *		free(buf);
- *	}
- *
  *	// Child has received fd, start reading loop.
  *	static struct io_plan *got_infd(struct io_conn *conn, int *infd)
  *	{
  *		struct buf *buf = calloc(1, sizeof(*buf));
- *		struct io_conn *new_conn;
  *
- *		new_conn = io_new_conn(NULL, *infd, read_more, buf);
- *		io_set_finish(new_conn, free_buf, buf);
+ *		io_new_conn(NULL, *infd, read_more, buf);
  *		return io_close(conn);
  *	}
  *	// Child is receiving the fd to read into. 

ccan/ccan/io/fdpass/fdpass.c

@@ -4,12 +4,6 @@
 #include <ccan/io/io_plan.h>
 #include <errno.h>
 
-static void destroy_conn_close_send_fd(struct io_conn *conn,
-				       struct io_plan_arg *arg)
-{
-	close(arg->u1.s);
-}
-
 static int do_fd_send(int fd, struct io_plan_arg *arg)
 {
 	if (!fdpass_send(fd, arg->u1.s)) {
@@ -18,11 +12,8 @@ static int do_fd_send(int fd, struct io_plan_arg *arg)
 			return 0;
 		return -1;
 	}
-	if (arg->u2.vp) {
-		struct io_conn *conn = arg->u2.vp;
+	if (arg->u2.s)
 		close(arg->u1.s);
-		tal_del_destructor2(conn, destroy_conn_close_send_fd, arg);
-	}
 	return 1;
 }
 
@@ -35,11 +26,7 @@ struct io_plan *io_send_fd_(struct io_conn *conn,
 	struct io_plan_arg *arg = io_plan_arg(conn, IO_OUT);
 
 	arg->u1.s = fd;
-	/* We need conn ptr for destructor */
-	arg->u2.vp = fdclose ? conn : NULL;
-	/* If conn closes before sending, we still need to close fd */
-	if (fdclose)
-		tal_add_destructor2(conn, destroy_conn_close_send_fd, arg);
+	arg->u2.s = fdclose;
 
 	return io_set_plan(conn, IO_OUT, do_fd_send, next, next_arg);
 }
@@ -52,9 +39,7 @@ static int do_fd_recv(int fd, struct io_plan_arg *arg)
 		/* In case ccan/io ever gets smart with non-blocking. */
 		if (errno == EAGAIN || errno == EWOULDBLOCK)
 			return 0;
-		/* If they can't handle the error, this will close conn! */
-		if (!io_get_extended_errors())
-			return -1;
+		return -1;
 	}
 	*(int *)arg->u1.vp = fdin;
 	return 1;