Add python to runtime docker image (arm32/64)

.dockerignore

@@ -1,3 +0,0 @@
-Dockerfile
-contrib/docker/Dockerfile.*
-target

.editorconfig

@@ -16,3 +16,6 @@ indent_size = 8
 indent_style = space
 indent_size = 4
 
+[.travis.yml]
+indent_style = space
+indent_size = 2

.gitattributes

@@ -7,14 +7,4 @@ configure text eol=lf
 # The following files are generated and should not be shown in the
 # diffs by default on Github.
 doc/lightning*.7 linguist-generated=true
-db_*_sqlgen.c linguist-generated=true
-statements_gettextgen.po linguist-generated=true
-*_wiregen.? linguist-generated=true
-*_printgen.? linguist-generated=true
 
-# The following are marked as binary and generated since they can be
-# easily overwritten and don't need deep review.
-cln-grpc/proto/node.proto -text -diff linguist-generated=true
-cln-grpc/src/convert.rs -text -diff linguist-generated=true
-cln-rpc/src/model.rs -text -diff linguist-generated=true
-contrib/pyln-testing/pyln/testing/node_pb2.py -text -diff linguist-generated=true

.github/CODEOWNERS

@@ -4,16 +4,15 @@
 # also can optionally require approval from a code owner before the
 # author can merge a pull request in the repository.
 
-cln-grpc/               @cdecker
-cln-rpc/                @cdecker
-plugins/src/            @cdecker
-plugins/grpc-plugin/    @cdecker
-contrib/reprobuild/     @cdecker
-contrib/msggen/         @cdecker
-contrib/pyln-client/    @cdecker
-contrib/pyln-testing/   @cdecker
-# Needed to ensure hsmd wire compatibility between releases
-hsmd/hsmd_wire.csv      @cdecker @ksedgwic @devrandom
+wallet/                 @cdecker
+*.py                    @cdecker
+
+wallet/invoices.*       @ZmnSCPxj
+
+common/param.*          @wythe
+common/json.*           @wythe
+common/json_tok.*       @wythe
+common/wallet_tx.*      @wythe
 
 # See https://help.github.com/articles/about-codeowners/ for more
 # information

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,16 +0,0 @@
-> [!IMPORTANT]
->
-> 25.09 FREEZE July 28TH: Non-bugfix PRs not ready by this date will wait for 25.12.
->
-> RC1 is scheduled on _August 11th_
->
-> The final release is scheduled for September 1st.
-
-
-## Checklist
-Before submitting the PR, ensure the following tasks are completed. If an item is not applicable to your PR, please mark it as checked:
-
-- [ ] The changelog has been updated in the relevant commit(s) according to the [guidelines](https://docs.corelightning.org/docs/coding-style-guidelines#changelog-entries-in-commit-messages).
-- [ ] Tests have been added or modified to reflect the changes.
-- [ ] Documentation has been reviewed and updated as needed.
-- [ ] Related issues have been listed and linked, including any that this PR closes.

.github/scripts/install-bitcoind.sh

@@ -1,31 +0,0 @@
-#!/bin/sh
-
-set -e
-
-export BITCOIN_VERSION=27.1
-export ELEMENTS_VERSION=23.2.1
-
-DIRNAME="bitcoin-${BITCOIN_VERSION}"
-EDIRNAME="elements-${ELEMENTS_VERSION}"
-FILENAME="${DIRNAME}-x86_64-linux-gnu.tar.gz"
-EFILENAME="${EDIRNAME}-x86_64-linux-gnu.tar.gz"
-
-cd /tmp/
-
-# Since we inadvertently broke `elementsd` support in the past we only
-# want to download and enable the daemon that is actually going to be
-# used when running in CI. Otherwise we could end up accidentally
-# testing against `bitcoind` but still believe that we ran against
-# `elementsd`.
-if [ "$TEST_NETWORK" = "liquid-regtest" ]; then
-    wget "https://github.com/ElementsProject/elements/releases/download/elements-${ELEMENTS_VERSION}/${EFILENAME}"
-    tar -xf "${EFILENAME}"
-    sudo mv "${EDIRNAME}"/bin/* "/usr/local/bin"
-    rm -rf "${EFILENAME}" "${EDIRNAME}"
-else
-    wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/${FILENAME}"
-    tar -xf "${FILENAME}"
-    sudo mv "${DIRNAME}"/bin/* "/usr/local/bin"
-    rm -rf "${FILENAME}" "${DIRNAME}"
-fi
-

.github/scripts/setup.sh

@@ -1,93 +0,0 @@
-#!/bin/bash
-set -e
-export DEBIAN_FRONTEND=noninteractive
-export RUST_VERSION=stable
-
-sudo useradd -ms /bin/bash tester
-sudo apt-get update -qq
-
-sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
-     autoconf \
-     automake \
-     binfmt-support \
-     build-essential \
-     clang \
-     cppcheck \
-     docbook-xml \
-     eatmydata \
-     gcc-aarch64-linux-gnu \
-     gcc-arm-linux-gnueabihf \
-     gcc-arm-none-eabi \
-     gettext \
-     git \
-     gnupg \
-     jq \
-     libc6-dev-arm64-cross \
-     libc6-dev-armhf-cross \
-     libev-dev \
-     libevent-dev \
-     libffi-dev \
-     libicu-dev \
-     libpq-dev \
-     libprotobuf-c-dev \
-     libsqlite3-dev \
-     libssl-dev \
-     libtool \
-     libxml2-utils \
-     locales \
-     net-tools \
-     postgresql \
-     python-pkg-resources \
-     python3 \
-     python3-dev \
-     python3-pip \
-     python3-setuptools \
-     qemu \
-     qemu-system-arm \
-     qemu-user-static \
-     shellcheck \
-     software-properties-common \
-     sudo \
-     tcl \
-     tclsh \
-     unzip \
-     valgrind \
-     wget \
-     xsltproc \
-     systemtap-sdt-dev \
-     zlib1g-dev
-
-echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
-sudo chmod 0440 /etc/sudoers.d/tester
-
-"$(dirname "$0")"/install-bitcoind.sh
-
-curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
-     -y --default-toolchain ${RUST_VERSION}
-
-# We also need a relatively recent protobuf-compiler, at least 3.12.0,
-# in order to support the experimental `optional` flag.
-
-# BUT WAIT!  Gentoo wants this to match the version from the Python protobuf,
-# which comes from the same tree.  Makes sense!
-
-# And
-#   grpcio-tools-1.69.0` requires `protobuf = ">=5.26.1,<6.0dev"`
-
-# Now, protoc changed to date-based releases, BUT Python protobuf
-# didn't, so Python protobuf 4.21.12 (in Ubuntu 23.04) corresponds to
-# protoc 21.12 (which, FYI, is packaged in Ubuntu as version 3.21.12).
-
-# In general protobuf version x.y.z corresponds to protoc version y.z
-
-# Honorable mention go to Matt Whitlock for spelunking this horror with me!
-
-PROTOC_VERSION=29.4
-PB_REL="https://github.com/protocolbuffers/protobuf/releases"
-curl -LO $PB_REL/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip
-sudo unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local/
-sudo chmod a+x /usr/local/bin/protoc
-export PROTOC=/usr/local/bin/protoc
-export PATH=$PATH:/usr/local/bin
-env
-ls -lha /usr/local/bin

.github/scripts/sync-rpc-cmds.py

@@ -1,115 +0,0 @@
-import os
-from time import sleep
-import requests
-import re
-from enum import Enum
-
-# readme url
-URL = "https://dash.readme.com/api/v1"
-# category id for API reference
-CATEGORY_ID = "63e4e160c60b2e001dd1cc4e"
-CATEGORY_SLUG = "json-rpc-apis"
-
-
-class Action(Enum):
-    ADD = 'add'
-    UPDATE = 'update'
-    DELETE = 'delete'
-
-
-def getListOfRPCDocs(headers):
-    response = requests.get(f"{URL}/categories/{CATEGORY_SLUG}/docs", headers=headers)
-    if response.status_code == 200:
-        return response.json()
-    else:
-        return []
-
-
-def publishDoc(action, title, body, order, headers):
-    payload = {
-        "title": title,
-        "type": "basic",
-        "body": body,
-        "category": CATEGORY_ID,
-        "hidden": False,
-        "order": order,
-    }
-    # title == slug
-    if action == Action.ADD:
-        # create doc
-        response = requests.post(URL + "/docs", json=payload, headers=headers)
-        if response.status_code != 201:
-            print(response.text)
-        else:
-            print("Created ", title)
-    elif action == Action.UPDATE:
-        # update doc
-        response = requests.put(f"{URL}/docs/{title}", json=payload, headers=headers)
-        if response.status_code != 200:
-            print(response.text)
-        else:
-            print("Updated ", title)
-    elif action == Action.DELETE:
-        # delete doc
-        response = requests.delete(f"{URL}/docs/{title}", headers=headers)
-        if response.status_code != 204:
-            print(response.text)
-        else:
-            print("Deleted ", title)
-    else:
-        print("Invalid action")
-
-
-def extract_rpc_commands(rst_content):
-    manpages_block = re.search(
-        r"\.\. block_start manpages(.*?)" r"\.\. block_end manpages",
-        rst_content,
-        re.DOTALL,
-    )
-    if manpages_block:
-        commands = re.findall(
-            r"\b([a-zA-Z0-9_-]+)" r"\s+<([^>]+)>\n", manpages_block.group(1)
-        )
-        return commands
-    return []
-
-
-def main():
-    # define headers for requests
-    headers = {
-        "accept": "application/json",
-        "content-type": "application/json",
-        "authorization": "Basic " + os.environ.get("README_API_KEY"),
-    }
-
-    # path to the rst file from where we fetch all the RPC commands
-    path_to_rst = "doc/index.rst"
-    with open(path_to_rst, "r") as file:
-        rst_content = file.read()
-
-    commands_from_local = extract_rpc_commands(rst_content)
-    commands_from_readme = getListOfRPCDocs(headers)
-
-    # Compare local and server commands list to get the list of command to add or delete
-    commands_local_title = set(command[0] for command in commands_from_local)
-    commands_readme_title = set(command['title'] for command in commands_from_readme)
-    commands_to_delete = commands_readme_title - commands_local_title
-    commands_to_add = commands_local_title - commands_readme_title
-    for name in commands_to_delete:
-        publishDoc(Action.DELETE, name, "", 0, headers)
-        sleep(3)
-
-    if commands_from_local:
-        order = 0
-        for name, file in commands_from_local:
-            with open("doc/" + file) as f:
-                body = f.read()
-                publishDoc(Action.ADD if name in commands_to_add else Action.UPDATE, name, body, order, headers)
-            order = order + 1
-            sleep(3)
-    else:
-        print("No commands found in the Manpages block.")
-
-
-if __name__ == "__main__":
-    main()

.github/workflows/bsd.yml

@@ -1,79 +0,0 @@
-name: FreeBSD Test
-
-on:
-  push:
-    branches:
-      - "master"
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  testfreebsd:
-    runs-on: ubuntu-22.04
-    name: Build and test on FreeBSD
-    timeout-minutes: 120
-    strategy:
-      fail-fast: true
-      matrix:
-        bitcoind-version: ["27.1"]
-    steps:
-    - uses: actions/checkout@v4
-    - name: Test in FreeBSD
-      id: test
-      uses: vmactions/freebsd-vm@v1
-      with:
-        usesh: true
-        sync: rsync
-        copyback: false
-        prepare: |
-          pkg install -y \
-            bash \
-            wget \
-            python310 \
-            gmake \
-            git \
-            python \
-            postgresql16-server \
-            autoconf \
-            automake \
-            libtool \
-            bash \
-            gettext \
-            sqlite3 \
-            lowdown \
-            pkgconf \
-            jq \
-            protobuf \
-            curl
-
-            python3.10 -m ensurepip
-            python3.10 -m pip install --upgrade pip
-            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
-
-
-        run: |
-          PATH=/root/.local/bin:$PATH:/root/.cargo/bin; export PATH
-          pip install --user -U wheel pip
-          pip3 install --user poetry
-          poetry install
-
-          git clone https://github.com/lightning/bolts.git ../bolts
-          # fatal: unsafe repository ('/Users/runner/work/lightning/lightning' is owned by someone else)
-          git config --global --add safe.directory `pwd`
-          for d in libsodium libwally-core gheap jsmn libbacktrace lowdown; do git config --global --add safe.directory `pwd`/external/$d; done
-          git submodule update --init --recursive
-
-          ./configure CC="$CC" --disable-valgrind
-
-          cat config.vars
-
-          cat << EOF > pytest.ini
-          [pytest]
-          addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
-          markers =
-              slow_test: marks tests as slow (deselect with '-m "not slow_test"')
-          EOF
-
-          # Just run a "quick" test without memory checking
-          poetry run gmake
-

.github/workflows/ci.yaml

@@ -1,653 +0,0 @@
----
-name: Continuous Integration
-on:
-  push:
-    branches:
-      - "master"
-  pull_request:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-env:
-  # Makes the upload-artifact work more reliably at the cost
-  # of a bit of compile time.
-  RUST_PROFILE: release
-  SLOW_MACHINE: 1
-  CI_SERVER_URL: "http://35.239.136.52:3170"
-
-jobs:
-  prebuild:
-    name: Pre-build checks
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    env:
-      BOLTDIR: bolts
-    strategy:
-      fail-fast: true
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
-
-      - name: Rebase
-        # We can't rebase if we're on master already.
-        if: github.ref != 'refs/heads/master'
-        run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-          git fetch origin ${{ github.base_ref }}
-          git rebase origin/${{ github.base_ref }}
-
-      - name: Check changelog
-        env:
-          PR_DESCRIPTION: "${{ github.event.pull_request.body || '' }}"
-          EVENT_NAME: "${{ github.event_name }}"
-          BASE_REF: "${{ github.base_ref || 'master' }}"
-        run: |
-          echo "Event Name: $EVENT_NAME"
-          echo "Base Ref: $BASE_REF"
-          echo "PR DESCRIPTION: $PR_DESCRIPTION"
-          if [ "$EVENT_NAME" = "pull_request" ]; then
-            if [[ "$PR_DESCRIPTION" != *"Changelog-None"* && \
-                  -z "$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')" && \
-                  "$(git rev-parse --abbrev-ref HEAD)" != "$BASE_REF" ]]; then
-              echo "::error::'Changelog' entry is missing in all commits, and 'Changelog-None' not specified in the PR description"
-              exit 1
-            else
-              if [[ "$PR_DESCRIPTION" == *"Changelog-None"* ]]; then
-                echo "Changelog found in PR description"
-              else
-                echo "Changelog found in Commit \"$(git log origin/$BASE_REF..HEAD --oneline --grep='Changelog-')\""
-              fi
-            fi
-          else
-            echo "Not a PR event."
-          fi
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          # Export and then use pip to install into the current env
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-          # We're going to check BOLT quotes, so get the latest version
-          git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
-      - name: Configure
-        run: ./configure --enable-debugbuild --enable-rust
-      - name: Check source
-        env:
-          VALGRIND: 0
-          PYTEST_OPTS: --timeout=1200
-        run: make check-source BASE_REF="origin/${{ github.base_ref }}"
-      - name: Check Generated Files have been updated
-        run: make check-gen-updated
-      - name: Check docs
-        run: make check-doc
-
-  compile:
-    name: Compile CLN ${{ matrix.cfg }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    needs:
-      - prebuild
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - CFG: compile-gcc
-            VALGRIND: 1
-            COMPILER: gcc
-          - CFG: compile-gcc-O3
-            VALGRIND: 1
-            COMPILER: gcc
-            COPTFLAGS_VAR: COPTFLAGS="-O3 -Werror"
-          # While we're at it let's try to compile with clang
-          - CFG: compile-clang
-            VALGRIND: 1
-            COMPILER: clang
-          - CFG: compile-clang-sanitizers
-            COMPILER: clang
-            ASAN: 1
-            UBSAN: 1
-            VALGRIND: 0
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-
-      - name: Build
-        env:
-          COMPILER: ${{ matrix.COMPILER }}
-          ASAN: ${{ matrix.ASAN }}
-          UBSAN: ${{ matrix.UBSAN }}
-          VALGRIND: ${{ matrix.VALGRIND }}
-          COMPAT: 1
-          CFG: ${{ matrix.CFG }}
-        run: |
-          set -e
-          pip3 install --user pip wheel poetry
-          poetry self add poetry-plugin-export
-          poetry export -o requirements.txt --with dev --without-hashes
-          python3 -m pip install -r requirements.txt
-          ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }}
-
-          make -j $(nproc) testpack.tar.bz2
-
-          # Rename now so we don't clash
-          mv testpack.tar.bz2 cln-${CFG}.tar.bz2
-      - name: Check rust packages
-        run: cargo test --all
-      - uses: actions/upload-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-          path: cln-${{ matrix.CFG }}.tar.bz2
-
-  check-units:
-    # The unit test checks are not in the critical path (not dependent
-    # on the integration tests), so run them with `valgrind`
-    name: Run unit tests
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    env:
-      BOLTDIR: bolts
-    needs:
-      - compile
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - CFG: compile-gcc
-            VALGRIND: 1
-          - CFG: compile-clang-sanitizers
-            VALGRIND: 0
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          sudo apt-get update -qq
-          sudo apt-get install -y -qq lowdown
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          # Export and then use pip to install into the current env
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-          # We're going to check BOLT quotes, so get the latest version
-          git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-
-      - name: Check
-        run: |
-          tar -xaf cln-${{ matrix.CFG }}.tar.bz2
-          eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }}
-
-  check-fuzz:
-    name: Run fuzz regression tests
-    runs-on: ubuntu-22.04
-    needs:
-      - prebuild
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          # Export and then use pip to install into the current env
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-
-      - name: Build
-        run: |
-          ./configure --enable-debugbuild --enable-fuzzing --enable-address-sanitizer --enable-ub-sanitizer --disable-valgrind CC=clang
-          make -j $(nproc) check-fuzz
-
-  integration:
-    name: Test CLN ${{ matrix.name }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    env:
-      RUST_PROFILE: release  # Has to match the one in the compile step
-      PYTEST_OPTS: --timeout=1200
-    needs:
-      - compile
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - NAME: gcc
-            CFG: compile-gcc
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: gcc
-            TEST_NETWORK: regtest
-          # While we're at it let's try to compile with clang
-          - NAME: clang
-            CFG: compile-clang
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: clang
-            TEST_NETWORK: regtest
-          # And of course we want to test postgres too
-          - NAME: postgres
-            CFG: compile-gcc
-            COMPILER: gcc
-            TEST_DB_PROVIDER: postgres
-            TEST_NETWORK: regtest
-          # And don't forget about elements (like cdecker did when
-          # reworking the CI...)
-          - NAME: liquid
-            CFG: compile-gcc
-            COMPILER: gcc
-            TEST_NETWORK: liquid-regtest
-            TEST_DB_PROVIDER: sqlite3
-          # And dual funding!
-          - NAME: dual-fund
-            CFG: compile-gcc
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: gcc
-            TEST_NETWORK: regtest
-            EXPERIMENTAL_DUAL_FUND: 1
-          # And splicing!
-          - NAME: splicing
-            CFG: compile-gcc
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: gcc
-            TEST_NETWORK: regtest
-            EXPERIMENTAL_SPLICING: 1
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          pip3 install --user pip wheel poetry
-          poetry install
-
-      - name: Install bitcoind
-        env:
-          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
-        run: .github/scripts/install-bitcoind.sh
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-
-      - name: Unpack pre-built CLN
-        env:
-          CFG: ${{ matrix.CFG }}
-        run: |
-          tar -xaf cln-${CFG}.tar.bz2
-
-      - name: Switch network
-        if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }}
-        run: |
-          # Loading the network from config.vars rather than the envvar is a terrible idea...
-          sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars
-          cat config.vars
-
-      - name: Test
-        env:
-          COMPILER: ${{ matrix.COMPILER }}
-          EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }}
-          EXPERIMENTAL_SPLICING: ${{ matrix.EXPERIMENTAL_SPLICING }}
-          COMPAT: 1
-          CFG: ${{ matrix.CFG }}
-          SLOW_MACHINE: 1
-          PYTEST_PAR: 10
-          TEST_DEBUG: 1
-          TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
-          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
-          LIGHTNINGD_POSTGRES_NO_VACUUM: 1
-        run: |
-          env
-          cat config.vars
-          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
-
-  integration-valgrind:
-    name: Valgrind Test CLN ${{ matrix.name }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    env:
-      RUST_PROFILE: release  # Has to match the one in the compile step
-      CFG: compile-gcc
-      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800
-    needs:
-      - compile
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - NAME: Valgrind (01/10)
-            PYTEST_OPTS: --test-group=1  --test-group-count=10
-          - NAME: Valgrind (02/10)
-            PYTEST_OPTS: --test-group=2  --test-group-count=10
-          - NAME: Valgrind (03/10)
-            PYTEST_OPTS: --test-group=3  --test-group-count=10
-          - NAME: Valgrind (04/10)
-            PYTEST_OPTS: --test-group=4  --test-group-count=10
-          - NAME: Valgrind (05/10)
-            PYTEST_OPTS: --test-group=5  --test-group-count=10
-          - NAME: Valgrind (06/10)
-            PYTEST_OPTS: --test-group=6  --test-group-count=10
-          - NAME: Valgrind (07/10)
-            PYTEST_OPTS: --test-group=7  --test-group-count=10
-          - NAME: Valgrind (08/10)
-            PYTEST_OPTS: --test-group=8  --test-group-count=10
-          - NAME: Valgrind (09/10)
-            PYTEST_OPTS: --test-group=9  --test-group-count=10
-          - NAME: Valgrind (10/10)
-            PYTEST_OPTS: --test-group=10 --test-group-count=10
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          sudo apt-get update -qq
-          sudo apt-get install -yyq valgrind
-          pip3 install --user pip wheel poetry
-          poetry install
-
-      - name: Install bitcoind
-        run: .github/scripts/install-bitcoind.sh
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-compile-gcc.tar.bz2
-
-      - name: Unpack build
-        run: tar -xvjf cln-compile-gcc.tar.bz2
-
-      - name: Test
-        env:
-          SLOW_MACHINE: 1
-          TEST_DEBUG: 1
-        run: |
-          VALGRIND=1 poetry run eatmydata pytest tests/ -vvv -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
-
-  integration-sanitizers:
-    name: Sanitizers Test CLN
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    env:
-      RUST_PROFILE: release
-      SLOW_MACHINE: 1
-      TEST_DEBUG: 1
-      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800
-    needs:
-      - compile
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - NAME: ASan/UBSan (01/10)
-            PYTEST_OPTS: --test-group=1  --test-group-count=10
-          - NAME: ASan/UBSan (02/10)
-            PYTEST_OPTS: --test-group=2  --test-group-count=10 -n 1
-          - NAME: ASan/UBSan (03/10)
-            PYTEST_OPTS: --test-group=3  --test-group-count=10
-          - NAME: ASan/UBSan (04/10)
-            PYTEST_OPTS: --test-group=4  --test-group-count=10
-          - NAME: ASan/UBSan (05/10)
-            PYTEST_OPTS: --test-group=5  --test-group-count=10
-          - NAME: ASan/UBSan (06/10)
-            PYTEST_OPTS: --test-group=6  --test-group-count=10
-          - NAME: ASan/UBSan (07/10)
-            PYTEST_OPTS: --test-group=7  --test-group-count=10
-          - NAME: ASan/UBSan (08/10)
-            PYTEST_OPTS: --test-group=8  --test-group-count=10
-          - NAME: ASan/UBSan (09/10)
-            PYTEST_OPTS: --test-group=9  --test-group-count=10
-          - NAME: ASan/UBSan (10/10)
-            PYTEST_OPTS: --test-group=10 --test-group-count=10
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          set -e
-          pip3 install --user wheel poetry
-          poetry install --with dev --no-root
-
-      - name: Install bitcoind
-        run: .github/scripts/install-bitcoind.sh
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-compile-clang-sanitizers.tar.bz2
-
-      - name: Unpack build
-        run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2
-
-      - name: Test
-        run: |
-          poetry run eatmydata pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
-
-  update-docs-examples:
-    name: Update examples in doc schemas (disabled temporarily!)
-    if: false
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-    env:
-      VALGRIND: 0
-      GENERATE_EXAMPLES: 1
-      PYTEST_OPTS: --timeout=1200
-      TEST_NETWORK: regtest
-    needs:
-      - compile
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          pip install -U pip wheel poetry
-          poetry self add poetry-plugin-export
-          poetry export -o /tmp/requirements.txt --without-hashes --with dev
-          pip install -r /tmp/requirements.txt
-      - name: Install bitcoind
-        env:
-          TEST_NETWORK: regtest
-        run: .github/scripts/install-bitcoind.sh
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-compile-gcc.tar.bz2
-      - name: Unpack pre-built CLN
-        run: |
-          tar -xaf cln-compile-gcc.tar.bz2
-      - name: Test
-        run: |
-          eatmydata make -j $(nproc) check-doc-examples
-
-  min-btc-support:
-    name: Test minimum supported BTC v${{ matrix.MIN_BTC_VERSION }} with ${{ matrix.NAME }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    env:
-      RUST_PROFILE: release  # Has to match the one in the compile step
-      PYTEST_OPTS: --timeout=1200
-    needs:
-      - compile
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - NAME: clang
-            CFG: compile-clang
-            TEST_DB_PROVIDER: sqlite3
-            COMPILER: clang
-            TEST_NETWORK: regtest
-            MIN_BTC_VERSION: '25.0'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          pip3 install --user pip wheel poetry
-          poetry install
-
-      - name: Download Bitcoin Core
-        run: wget "https://bitcoincore.org/bin/bitcoin-core-${{ matrix.MIN_BTC_VERSION }}/bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz"
-
-      - name: Extract Bitcoin Core
-        run: tar -xf "bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz"
-
-      - name: Move Bitcoin Core Binaries
-        run: sudo mv bitcoin-${{ matrix.MIN_BTC_VERSION }}/bin/* /usr/local/bin/
-
-      - name: Clean Up Downloaded Bitcoin
-        run: rm -rf "bitcoin-${{ matrix.MIN_BTC_VERSION }}-x86_64-linux-gnu.tar.gz" "bitcoin-${{ matrix.MIN_BTC_VERSION }}"
-
-      - name: Download build
-        uses: actions/download-artifact@v4
-        with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-
-      - name: Unpack pre-built CLN
-        env:
-          CFG: ${{ matrix.CFG }}
-        run: |
-          tar -xaf cln-${CFG}.tar.bz2
-
-      - name: Test
-        env:
-          COMPILER: ${{ matrix.COMPILER }}
-          COMPAT: 1
-          CFG: ${{ matrix.CFG }}
-          SLOW_MACHINE: 1
-          PYTEST_PAR: 10
-          TEST_DEBUG: 1
-          TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
-          TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
-          LIGHTNINGD_POSTGRES_NO_VACUUM: 1
-        run: |
-          env
-          cat config.vars
-          VALGRIND=0 poetry run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
-
-  check-flake:
-    name: Check Nix Flake
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: true
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: 'recursive'
-      - name: Check Nix flake inputs
-        uses: DeterminateSystems/flake-checker-action@v8
-      - name: Install Nix
-        uses: cachix/install-nix-action@V27
-        with:
-          nix_path: nixpkgs=channel:nixos-24.05
-      - name: Check flake
-        run: nix flake check .?submodules=1#
-
-  gather:
-    # A dummy task that depends on the full matrix of tests, and
-    # signals successful completion. Used for the PR status to pass
-    # before merging.  Needs to run even if they failed!
-    name: CI completion
-    runs-on: ubuntu-22.04
-    needs:
-      - integration
-      - check-units
-      - integration-valgrind
-      - integration-sanitizers
-      - min-btc-support
-      - check-flake
-    if: ${{ always() }}
-    steps:
-      - name: Complete
-        env:
-          JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC FLAKE"
-          INTEGRATION: ${{ needs.integration.result }}
-          CHECK_UNITS: ${{ needs['check-units'].result }}
-          VALGRIND: ${{ needs['integration-valgrind'].result }}
-          SANITIZERS: ${{ needs['integration-sanitizers'].result }}
-          DOCS: ${{ needs['update-docs-examples'].result }}
-          BTC: ${{ needs['min-btc-support'].result }}
-          FLAKE: ${{ needs['check-flake'].result }}
-        run: |
-          failed=""
-          for name in $JOB_NAMES; do
-            result="${!name}"
-            echo "$name: $result"
-            if [[ "$result" != "success" ]]; then
-              failed="yes"
-            fi
-          done
-          if [[ "$failed" == "yes" ]]; then
-            echo "One or more required jobs failed"
-            exit 1
-          fi

.github/workflows/crate-io.yml

@@ -1,30 +0,0 @@
----
-name: "Release Rust 🦀"
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - "master"
-
-jobs:
-  release_rust:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@stable
-      - uses: arduino/setup-protoc@v3
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './cln-rpc'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './plugins'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true
-      - uses: katyo/publish-crates@v2
-        with:
-          path: './cln-grpc'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-unpublished-changes: true

.github/workflows/docker-release.yml

@@ -1,118 +0,0 @@
-name: Build and push multi-platform docker images
-
-on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      version:
-        description: 'Release version'
-        required: true
-
-      repository-name:
-        description: 'Docker repository name'
-        default: 'elementsproject'
-        required: false
-
-      platforms-to-build:
-        description: 'List of platforms to build'
-        default: 'linux/amd64,linux/arm64,linux/arm/v7'
-        required: false
-  
-      push-latest:
-        description: 'Push the latest tag also?'
-        default: 'false'
-        required: false
-
-jobs:
-  build:
-    runs-on: ubuntu-22.04
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-      with:
-        ref: ${{ github.ref }}  # Ensures the branch triggering the workflow is checked out
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-
-    - name: Setup Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
-    - name: Log in to Docker Hub
-      uses: docker/login-action@v3
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
-    - name: Set up values
-      id: set-values
-      run: |
-        if [[ "${{ github.event.inputs.version }}" != "" ]]; then
-          echo "Input version provided"
-          VERSION=${{ github.event.inputs.version }}
-        elif [[ ${{ github.ref_type }} == "tag" ]]; then
-          echo "This is a tag event"
-          VERSION=${{ github.ref_name }}
-        else
-          echo "No version provided and no tag found."
-          exit 1
-        fi
-        echo "VERSION=$VERSION" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.repository-name }}" != "" ]]; then
-          REPONAME=${{ github.event.inputs.repository-name }}
-        else
-          REPONAME="elementsproject"
-        fi
-        echo "REPONAME=$REPONAME" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.platforms-to-build }}" != "" ]]; then
-          PLATFORMS=${{ github.event.inputs.platforms-to-build }}
-        else
-          PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
-        fi
-        echo "PLATFORMS=$PLATFORMS" >> $GITHUB_ENV
-
-        if [[ "${{ github.event.inputs.push-latest }}" == "true" ]] || 
-          ([[ "${{ github.ref_type }}" == "tag" ]] && [[ ! "$VERSION" =~ rc ]]); then
-          echo "Latest true"
-          PUSHLATEST="true"
-        else
-          echo "Latest false"
-          PUSHLATEST="false"
-        fi
-        echo "PUSHLATEST=$PUSHLATEST" >> $GITHUB_ENV
-
-        TAGS="$REPONAME/lightningd:$VERSION"
-        if [[ "$PUSHLATEST" == "true" ]]; then
-          TAGS="$TAGS,$REPONAME/lightningd:latest"
-        fi
-        echo "TAGS=$TAGS" >> $GITHUB_ENV
-
-    - name: Print GitHub Ref Values
-      run: |
-        echo "GITHUB REF TYPE: ${{ github.ref_type }}"
-        echo "GITHUB REF NAME: ${{ github.ref_name }}"
-        echo "EVENT INPUT VERSION: ${{ github.event.inputs.version }}"
-        echo "EVENT INPUT REPO: ${{ github.event.inputs.repository-name }}"
-        echo "EVENT INPUT PLATFORMS: ${{ github.event.inputs.platforms-to-build }}"
-        echo "EVENT INPUT PUSH LATEST: ${{ github.event.inputs.push-latest }}"
-        echo "ENV VERSION: ${{ env.VERSION }}"
-        echo "ENV REPO NAME: ${{ env.REPONAME }}"
-        echo "ENV PLATFORMS: ${{ env.PLATFORMS }}"
-        echo "ENV PUSH LATEST: ${{ env.PUSHLATEST }}"
-        echo "ENV TAGS: ${{ env.TAGS }}"
-
-    - name: Build and push Docker image
-      uses: docker/build-push-action@v5
-      with:
-        context: .
-        file: ./Dockerfile
-        push: true
-        platforms: ${{ env.PLATFORMS }}
-        tags: ${{ env.TAGS }}

.github/workflows/docker.yml

@@ -1,34 +0,0 @@
-name: Docker packaging
-on:
-  push:
-    tags:
-      - 'basedon-*'
-
-jobs:
-  main:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Environment variables
-        run: env
-      -
-        name: Create images
-        env:
-          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
-          DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }}
-          DOCKERHUB_REPO: ${{ vars.DOCKERHUB_REPO }}
-        shell: bash
-        run: |
-          LATEST_TAG=${GITHUB_REF#refs/tags/}
-          LATEST_TAG=${LATEST_TAG:8} #trim "basedon-" from tag
-          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USER" --password-stdin
-          docker buildx create --use
-          DOCKER_BUILDX_OPTS="--platform linux/amd64,linux/arm64,linux/arm/v7 --push"
-          docker buildx build $DOCKER_BUILDX_OPTS -t $DOCKERHUB_REPO:$LATEST_TAG .

.github/workflows/macos.yaml

@@ -1,62 +0,0 @@
----
-name: Mac OS pytest
-on:
-  pull_request:
-jobs:
-  smoke-test:
-    name: Smoke Test macOS
-    runs-on: macos-14
-    timeout-minutes: 120
-    strategy:
-      fail-fast: true
-      matrix:
-        bitcoind-version: ["27.1"]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Download Bitcoin ${{ matrix.bitcoind-version }} & install binaries
-        run: |
-          export BITCOIND_VERSION=${{ matrix.bitcoind-version }}
-          export TARGET_ARCH="arm64-apple-darwin"
-
-          wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
-          tar -xzf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz
-          sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin
-          rm -rf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz bitcoin-${BITCOIND_VERSION}
-
-      - name: Install dependencies
-        run: |
-          export PATH="/usr/local/opt:/Users/runner/.local/bin:/opt/homebrew/bin/python3.10/bin:$PATH"
-
-          brew install gnu-sed python@3.10 autoconf automake libtool protobuf
-          python3.10 -m pip install -U --user poetry==1.8.0 wheel pip mako
-          python3.10 -m poetry install
-
-      - name: Build and install CLN
-        run: |
-          export CPATH=/opt/homebrew/include
-          export LIBRARY_PATH=/opt/homebrew/lib
-
-          python3.10 -m poetry run ./configure --disable-valgrind --disable-compat
-          python3.10 -m poetry run make
-
-      - name: Start bitcoind in regtest mode
-        run: |
-          bitcoind -regtest -daemon
-          sleep 5
-
-      - name: Generate initial block
-        run: |
-          bitcoin-cli -regtest createwallet default_wallet
-          bitcoin-cli -regtest generatetoaddress 1 $(bitcoin-cli -regtest getnewaddress)
-          sleep 2
-
-      - name: Start CLN in regtest mode
-        run: |
-          lightningd/lightningd --network=regtest --log-file=/tmp/l1.log --daemon
-          sleep 5
-
-      - name: Verify CLN is running
-        run: |
-          cli/lightning-cli --regtest getinfo

.github/workflows/pypi.yml

@@ -1,101 +0,0 @@
-name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
-
-on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      dist-location:
-        description: 'Distribution location (test/prod)'
-        default: 'test'
-        required: false
-
-jobs:
-  deploy:
-    name: Build and publish ${{ matrix.package }} 🐍
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - PACKAGE: pyln-client
-            WORKDIR: contrib/pyln-client
-          - PACKAGE: pyln-testing
-            WORKDIR: contrib/pyln-testing
-          - PACKAGE: pyln-proto
-            WORKDIR: contrib/pyln-proto
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-      with:
-        # Need to fetch entire history in order to locate the version tag
-        fetch-depth: 0
-
-    - name: Check version tag
-      run: >-
-        git describe --tags --always --dirty=-modded --abbrev=7
-
-    - name: Setup Version
-      env:
-        WORKDIR: ${{ matrix.WORKDIR }}
-      run: |
-        echo "VERSION=$(git describe --tags --abbrev=0).post$(git describe --tags --abbrev=1 | awk -F "-" '{print $2}')" >> $GITHUB_ENV
-
-    - name: Set up values
-      id: set-values
-      run: |
-        if [[ "${{ github.event.inputs.dist-location }}" != "" ]]; then
-          DISTLOCATION=${{ github.event.inputs.dist-location }}
-        elif [[ "${{ github.ref_type }}" == "tag" ]] && [[ ! "${{ github.ref_name }}" =~ rc ]]; then
-          DISTLOCATION="prod"
-        else
-          DISTLOCATION="test"
-        fi
-        echo "DISTLOCATION=$DISTLOCATION" >> $GITHUB_OUTPUT
-        echo "EVENT DISTLOCATION: ${{ github.event.inputs.dist-location }}"
-        echo "DISTRIBUTION LOCATION: $DISTLOCATION"
-  
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.10'
-
-    - name: Install Poetry
-      run: |
-        curl -sSL https://install.python-poetry.org | python3 -
-        echo "$HOME/.local/bin" >> $GITHUB_PATH
-        echo "PATH=$HOME/.local/bin:$PATH"
-
-    - name: Publish distribution 📦 to Test PyPI
-      if: github.repository == 'ElementsProject/lightning' && steps.set-values.outputs.DISTLOCATION == 'test'
-      env:
-        POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.TEST_PYPI_API_TOKEN }}
-        WORKDIR: ${{ matrix.WORKDIR }}
-      run: |
-        echo "POETRY VERSION TEST: $(poetry --version)"
-        echo "Pyln VERSION: $VERSION"
-        cd ${{ env.WORKDIR }}
-        python3 -m pip config set global.timeout 150
-        poetry config repositories.testpypi https://test.pypi.org/legacy/
-        make upgrade-version NEW_VERSION=$VERSION
-        poetry build  --no-interaction
-        poetry publish --repository testpypi --no-interaction --skip-existing
-
-    - name: Publish distribution 📦 to PyPI
-      if: github.repository == 'ElementsProject/lightning' && steps.set-values.outputs.DISTLOCATION == 'prod'
-      env:
-        POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
-        WORKDIR: ${{ matrix.WORKDIR }}
-      run: |
-        echo "POETRY VERSION PUBLISH: $(poetry --version)"
-        cd ${{ env.WORKDIR }}
-        export VERSION=$(git describe --tags --abbrev=0)
-        echo "Pyln VERSION: $VERSION"
-        make upgrade-version NEW_VERSION=$VERSION
-        python3 -m pip config set global.timeout 150
-        poetry build  --no-interaction
-        poetry publish --no-interaction

.github/workflows/rdme-docs-sync.yml

@@ -1,51 +0,0 @@
-name: ReadMe Sync
-
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'doc/**'
-  workflow_dispatch:
-
-jobs:
-  rdme-docs-sync:  
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Check out repo 📚
-        uses: actions/checkout@v4
-
-      - name: Sync doc/getting-started/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/getting-started --key=${{ env.README_API_KEY }} --version=1
-            
-      - name: Sync doc/beginners-guide/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/beginners-guide --key=${{ env.README_API_KEY }} --version=1
-      
-      - name: Sync doc/node-operators-guide/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/node-operators-guide --key=${{ env.README_API_KEY }} --version=1
-      
-      - name: Sync doc/developers-guide/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/developers-guide --key=${{ env.README_API_KEY }} --version=1
-      
-      - name: Sync doc/contributing-to-core-lightning/ 🚀
-        uses: readmeio/rdme@v8
-        env:
-          README_API_KEY: ${{ secrets.README_API_KEY }}
-        with:
-          rdme: docs doc/contribute-to-core-lightning --key=${{ env.README_API_KEY }} --version=1

.github/workflows/readme-rpc-sync.yml

@@ -1,43 +0,0 @@
-name: ReadMe RPC Sync
-
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'doc/schemas/*.json'
-      - 'doc/*.1.md'
-      - 'doc/*.5.md'
-      - 'doc/*.8.md'
-      - 'doc/lightningd-*.7.md'
-      - 'doc/reckless.7.md'
-  workflow_dispatch:
-
-jobs:
-  rdme-rpc-sync:
-    runs-on: ubuntu-22.04
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
-
-    - name: Install python modules
-      run: |
-        python -m pip install requests mako grpcio-tools
-
-    - name: Install dependencies
-      run: bash -x .github/scripts/setup.sh
-
-    - name: Build (including rpc .md files)
-      run: |
-        ./configure --enable-debugbuild
-        make -j $(nproc)
-
-    - name: Set environment variable and run
-      env:
-        README_API_KEY: ${{ secrets.README_API_KEY }}
-      run: python .github/scripts/sync-rpc-cmds.py

.github/workflows/release.yml

@@ -1,191 +0,0 @@
----
-# https://docs.corelightning.org/docs/release-checklist
-name: "Release 🚀"
-on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+[0-9a-z]+'
-  workflow_dispatch:
-    inputs:
-      version:
-        description: 'Release version'
-        required: true
-      create_release:
-        description: Create a draft release
-        default: no
-        type: choice
-        options:
-          - yes
-          - no
-
-jobs:
-  check:
-    name: Check
-    outputs:
-      version: ${{ steps.capture.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.ref }}
-          fetch-depth: 0
-          fetch-tags: true
-
-      - name: Determine version
-        run: |
-          if [[ "${{ github.event.inputs.version }}" != "" ]]; then
-            VERSION="${{ github.event.inputs.version }}"
-          elif [ "${{ github.ref_type }}" == "tag" ]; then
-            VERSION="${{ github.ref_name }}"
-          else
-            echo "No release version provided and no tag found."
-            exit 1
-          fi
-          echo "VERSION=$VERSION" >> "$GITHUB_ENV"
-          echo "Determined version: $VERSION"
-
-      - name: Validate release
-        run: tools/check-release.sh --version=${VERSION}
-
-      - name: Catpure version output
-        id: capture
-        run: echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
-
-  releases:
-    name: Releases
-    needs: check
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false    # Let each build finish.
-      matrix:
-        target:
-          - 'bin-Fedora'
-          - 'bin-Ubuntu-focal'
-          - 'bin-Ubuntu-jammy'
-          - 'bin-Ubuntu-noble'
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-tags: true
-
-      # tools/build-release.sh requires lowdown
-      - name: Prepare base environment
-        run: |
-          sudo apt-get install -y lowdown
-          ./configure
-
-      - name: Build environment setup
-        run: |
-          distribution=$(echo ${{ matrix.target }} | cut -d'-' -f3)
-          echo "Building base image for ${distribution}"
-          sudo docker run --rm -v $(pwd):/build ubuntu:${distribution} bash -c "\
-              apt-get update && \
-              apt-get install -y debootstrap && \
-              debootstrap ${distribution} /build/${distribution}"
-          sudo tar -C ${distribution} -c . | docker import - ${distribution}
-
-          # Build Docker image
-          docker build -t cl-repro-${distribution} - < contrib/reprobuild/Dockerfile.${distribution}
-        if: contains(matrix.target, 'Ubuntu')
-
-      - name: Build release
-        run: tools/build-release.sh ${{ matrix.target }}
-
-      - name: Upload target artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          path: release/
-          name: ${{ matrix.target }}
-          if-no-files-found: error
-
-  artifact:
-    name: Construct release artifact
-    needs:
-      - check
-      - releases
-    env:
-      version: ${{ needs.check.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Merge artifacts
-        uses: actions/upload-artifact/merge@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          pattern: bin-*
-          delete-merged: true
-
-  release:
-    name: Sign and prepare release draft
-    needs:
-      - check
-      - artifact
-    env:
-      version: ${{ needs.check.outputs.version }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-tags: true
-
-      - name: Download artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          path: release/
-
-      - name: Import GPG keys
-        id: gpg
-        uses: crazy-max/ghaction-import-gpg@v6
-        with:
-          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.GPG_PASSPHRASE }}
-          trust_level: 5
-
-      - name: Set default GPG key
-        run: echo "default-key ${{ steps.gpg.outputs.keyid }}" >> ~/.gnupg/gpg.conf
-
-      - name: Sign release
-        run: |
-          sudo apt-get install -y lowdown
-          ./configure
-          tools/build-release.sh --without-zip sign
-          mv release/SHA256SUMS.asc${{ steps.gpg.outputs.keyid  }} release/SHA256SUMS.asc
-
-      - name: Upload signed artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: c-lightning-${{ env.version }}
-          overwrite: true
-          path: release/
-
-      - name: Determine release data
-        id: release_data
-        run: |
-          VERSION=${{ env.version }}
-          CHANGELOG_VERSION=${VERSION#v}
-          echo "CHANGELOG_VERSION=$CHANGELOG_VERSION"
-          echo "changelog_version=$CHANGELOG_VERSION" >> "$GITHUB_OUTPUT"
-
-          CHANGELOG_TITLE=$(grep "## \[${CHANGELOG_VERSION}\]" CHANGELOG.md)
-          echo "CHANGELOG_TITLE=$CHANGELOG_TITLE"
-          echo "changelog_title=$CHANGELOG_TITLE" >> "$GITHUB_OUTPUT"
-
-          RELEASE_TITLE=$(echo $CHANGELOG_TITLE | cut -d'"' -f2)
-          echo "RELEASE_TITLE=$RELEASE_TITLE"
-          echo "release_title=$RELEASE_TITLE" >> "$GITHUB_OUTPUT"
-
-      - name: Prepare release draft
-        if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.create_release == 'yes')
-        uses: softprops/action-gh-release@v2
-        with:
-          name: "${{ env.version }} ${{ steps.release_data.outputs.release_title }}"
-          tag_name: ${{ env.version }}
-          draft: true
-          prerelease: contains(env.version, "-rc")
-          files: release/*
-          fail_on_unmatched_files: true

.github/workflows/repro.yml

@@ -1,113 +0,0 @@
----
-# https://docs.corelightning.org/docs/repro
-name: Repro Build Nightly
-on:
-  # 05:00 Berlin, 03:00 UTC, 23:00 New York, 20:00 Los Angeles
-  schedule:
-    - cron: "0 3 * * *"
-  workflow_dispatch:
-
-jobs:
-  ubuntu:
-    name: "Ubuntu repro build: ${{ matrix.version }}"
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: false    # Let each build finish.
-      matrix:
-        version: ['focal', 'jammy', 'noble']
-    steps:
-      - name: Git checkout
-        uses: actions/checkout@v4
-
-      - name: Build environment setup - ${{ matrix.version }}
-        run: |
-          echo "Building base image for ${{ matrix.version }}"
-          echo "STEP=Build environment setup" >> "$GITHUB_ENV"
-          sudo docker run --rm -v $(pwd):/build ubuntu:${{ matrix.version }} bash -c "\
-            apt-get update && \
-            apt-get install -y debootstrap && \
-            debootstrap ${{ matrix.version }} /build/${{ matrix.version }}"
-          sudo tar -C ${{ matrix.version }} -c . | docker import - ${{ matrix.version }}
-
-      - name: Builder image setup - ${{ matrix.version }}
-        run: |
-          echo "STEP=Builder image setup" >> "$GITHUB_ENV"
-          docker build -t cl-repro-${{ matrix.version }} - < contrib/reprobuild/Dockerfile.${{ matrix.version }}
-
-      - name: Build reproducible image and store Git state - ${{ matrix.version }}
-        run: |
-          echo "STEP=Build reproducible image and store Git state" >> "$GITHUB_ENV"
-
-          # Create release directory.
-          mkdir $GITHUB_WORKSPACE/release
-
-          # Perform the repro build.
-          docker run --name cl-build -v $GITHUB_WORKSPACE:/repo -e FORCE_MTIME=$(date +%F) -t cl-repro-${{ matrix.version }}
-
-          # Commit the image in order to inspect the build later.
-          docker commit cl-build cl-repro
-
-          # Inspect the version.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "make version > /repo/release/version.txt"
-
-          # Inspect the Git tree state.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "\
-              git --no-pager status > /repo/release/git.log && \
-              git --no-pager diff >> /repo/release/git.log"
-
-          # Change permissions on the release files for access by the runner environment.
-          sudo chown -R runner $GITHUB_WORKSPACE/release
-
-      - name: Assert clean version - ${{ matrix.version }}
-        run: |
-          echo "STEP=Assert clean version" >> "$GITHUB_ENV"
-          echo 'Version:'
-          cat release/version.txt
-          echo -e
-
-          releasefile=$(ls release/clightning-*)
-          echo 'Release file:'
-          ls -al release/clightning-*
-          echo -e
-          if [ -n "$(cat release/version.txt | sed -n '/-modded/p')" ] || \
-             [ -n "$(echo $releasefile | sed -n '/-modded/p')" ]; then
-            echo "Git Status and Diff:"
-            cat release/git.log
-            echo -e
-            echo 'Error: release modded / dirty tree.'
-            exit 1
-          else
-            echo 'Success! Clean release.'
-          fi
-
-      - name: Upload release artifact - ${{ matrix.version }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: release-${{ matrix.version }}
-          path: release
-          retention-days: 3 # Automatically delete after 3 days
-
-      - name: Send email on failure
-        if: ${{ failure() }}
-        uses: dawidd6/action-send-mail@v3
-        with:
-          server_address: smtp.gmail.com
-          server_port: 587
-          username: ${{ secrets.EMAIL_USERNAME }}
-          password: ${{ secrets.EMAIL_PASSWORD }}
-          from: ${{ secrets.EMAIL_USERNAME }}
-          to: ${{ vars.DISTRIBUTION_LIST }}
-          subject: "CI Failure: Step ${{ env.STEP }} failed for distro ${{ matrix.version }}"
-          convert_markdown: true
-          html_body: |
-            <html>
-              <body>
-                <p>GitHub Workflow ${{ github.workflow }} Failed! For more details, click on the action below.</p>
-                <strong>Failure Details:</strong><br/>
-                <strong>Event: </strong>${{ github.event_name }}<br/>
-                <strong>Job: </strong>${{ github.job }}<br/>
-                <strong>Distro: </strong>${{ matrix.version }}<br/>
-                <strong>Step: </strong>${{ env.STEP }}<br/>
-                <strong>Action: </strong><a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}</a><br/>
-              </body>
-            </html>

.gitignore

@@ -8,42 +8,39 @@
 *.gcno
 *.dSYM
 *.rej
+*.po
 *.pyc
-*.tmp
-libccan.a
 .cppcheck-suppress
 .mypy_cache
 TAGS
 tags
 ccan/tools/configurator/configurator
 ccan/ccan/cdump/tools/cdump-enumstr
-*_gen.c
-*_gen.h
-gen_*.c
-gen_*.h
-wire/gen_*_csv
+gen_*
 cli/lightning-cli
+tools/check-bolt
+tools/hsmtool
+tools/lightning-hsmtool
 coverage
 ccan/config.h
 __pycache__
 config.vars
-monkeytype.sqlite3
 
 # Ignore some generated binaries
 */test/run-*
 !*/test/run-*.c
-*/test/exp-run-*
-!*/test/exp-run-*.c
-*/*/test/run-*
-!*/*/test/run-*.c
+external/libbacktrace-build/
+external/libbacktrace.a
+external/libbacktrace.la
+external/libbase58/
+external/libbase58.a
 test/test_protocol
 test/test_sphinx
 tests/.pytest.restart
 tests/plugins/test_libplugin
-tests/fuzz/fuzz-*
-!tests/fuzz/fuzz-*.c
 gossip_store
 .pytest_cache
+tools/headerversions
 .tmp.lightningrfc/
 contrib/pylightning/build/
 contrib/pylightning/dist/
@@ -51,54 +48,5 @@ contrib/pylightning/pylightning.egg-info/
 contrib/pyln-*/build/
 contrib/pyln-*/dist/
 contrib/pyln-*/pyln_*.egg-info/
-contrib/pyln-*/.eggs/
-contrib/pyln-*/pyln/*/__version__.py
-tests/plugins/test_selfdisable_after_getmanifest
-
-# Ignore generated files
-devtools/features
-doc/schemas/sql.json
-doc/*.7.md
-doc/*.[1578]
-doc/reckless*.[1578]
-*_sqlgen.[ch]
-*_wiregen.[ch]
-*_printgen.[ch]
-*_gettextgen.po
-tests/node_pb2.py
-tests/node_pb2_grpc.py
-tests/primitives_pb2.py
-tests/primitives_pb2_grpc.py
-tests/autogenerate-examples-status.log
-tests/autogenerate-examples.json
-tests/autogenerate-examples-repeat.log
-
-# Ignore unrelated stuff
-.DS_Store
-.gdb_history
-.python-version
-compile_commands.json
-
-# Rust targets
-target
-plugins/cln-grpc
-plugins/clnrest
-
-# Build directories
-bionic/
-focal/
-jammy/
-noble/
+plugins/keysend
 release/
-.vscode/
-.cache/
-
-# Ignore release verification Sha256Sums
-SHA256SUMS-*
-doc/.doc_version
-autogenerate-examples-status.log
-
-# Ignore nix outputs
-result
-result-[0-9]*
-tests/plugins/channeld_fakenet

.gitlab-ci.yml

@@ -10,7 +10,7 @@ build:
       curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN"
       -X POST
       --data '{"state": "pending", "description": "Gitlab-CI is building the commit", "context": "gitlab-ci"}'
-      https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_COMMIT_SHA || true
+      https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_BUILD_REF || true
   script:
     - make
     - make -j 12 check
@@ -27,7 +27,7 @@ update-status-fail:
   when: on_failure
   script:
     - >-
-      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "failure", "description": "Gitlab-CI build failed, please contact @cdecker for details about build #$CI_JOB_ID.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_COMMIT_SHA || true
+      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "failure", "description": "Gitlab-CI build failed, please contact @cdecker for details about build #$CI_BUILD_ID.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_BUILD_REF || true
 
 update-status-success:
   image: tutum/curl
@@ -35,4 +35,4 @@ update-status-success:
   when: on_success
   script:
     - >-
-      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "success", "description": "Gitlab-CI build succeeded.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_COMMIT_SHA || true
+      curl -s -H "Authorization: token $GITHUB_STATUS_TOKEN" -X POST --data '{"state": "success", "description": "Gitlab-CI build succeeded.", "context": "gitlab-ci"}' https://api.github.com/repos/ElementsProject/lightning/statuses/$CI_BUILD_REF || true

.gitmodules

@@ -11,10 +11,3 @@
 	path = external/libwally-core
 	url = https://github.com/ElementsProject/libwally-core.git
 	ignore = dirty
-[submodule "external/gheap"]
-	path = external/gheap
-	url = https://github.com/valyala/gheap
-[submodule "external/lowdown"]
-	path = external/lowdown
-	url = https://github.com/kristapsdz/lowdown.git
-	ignore = dirty

.pre-commit-config.yaml

@@ -1,13 +0,0 @@
-repos:
-- repo: https://github.com/astral-sh/ruff-pre-commit
-  # Ruff version.
-  rev: v0.8.0
-  hooks:
-    # Run the linter.
-    - id: ruff
-      args: [ --diff ]
-      exclude: "contrib/pyln-grpc-proto/pyln/grpc/(primitives|node)_pb2(|_grpc).py"
-    # Run the formatter.
-    - id: ruff-format
-      args: [ --diff ]
-      exclude: "contrib/pyln-grpc-proto/pyln/grpc/(primitives|node)_pb2(|_grpc).py"

.travis.yml

@@ -0,0 +1,47 @@
+language: c
+dist: bionic
+
+notifications:
+  email: false
+
+before_install:
+  sudo apt-get install -y libsqlite3-dev cppcheck valgrind gcc-4.8 gettext libc6-dev-armhf-cross libc6-dev-arm64-cross gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf qemu-user-static binfmt-support libsecp256k1-dev
+
+env:
+  # These run superficial checks that don't take too long on their own
+  - ARCH=64 TEST_CMD="make check-source check-units installcheck" COPTFLAGS="-O3"
+  - ARCH=32 TEST_CMD="make check-source check-units installcheck" DEVELOPER=0
+  - ARCH=64 TEST_CMD="make check-source check-units installcheck" DEVELOPER=0 COMPILER=gcc-4.8
+  - ARCH=64 TEST_CMD="make check-source check-units installcheck" DEVELOPER=1 EXPERIMENTAL_FEATURES=1
+
+  # All of the following will just run `make pytest`
+  - VALGRIND=0 ARCH=64 DEVELOPER=1 COMPILER=clang
+  - VALGRIND=0 ARCH=64 DEVELOPER=1 COMPILER=gcc
+  - VALGRIND=0 ARCH=64 DEVELOPER=0 COMPILER=gcc COMPAT=0 TEST_GROUP=1 TEST_GROUP_COUNT=2
+  - VALGRIND=0 ARCH=64 DEVELOPER=0 COMPILER=gcc COMPAT=0 TEST_GROUP=2 TEST_GROUP_COUNT=2
+
+  ## All of these are pretty slow since they run under valgrind
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=3 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=4 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=5 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=6 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=7 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=8 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=9 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=10 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=11 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=12 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=13 TEST_GROUP_COUNT=14
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=14 TEST_GROUP_COUNT=14
+  - VALGRIND=0 ARCH=arm32v7 DEVELOPER=1 TARGET_HOST=arm-linux-gnueabihf
+  - VALGRIND=0 ARCH=arm64v8 DEVELOPER=1 TARGET_HOST=aarch64-linux-gnu
+cache:
+  directories:
+    - external/x86_64-linux-gnu
+    - external/aarch64-linux-gnu
+    - external/arm-linux-gnueabihf
+
+script:
+  .travis/build.sh

.travis/build.sh

@@ -0,0 +1,117 @@
+#!/bin/bash -x
+set -e
+
+CWD=$(pwd)
+export SLOW_MACHINE=1
+export CC=${COMPILER:-gcc}
+export DEVELOPER=${DEVELOPER:-1}
+export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0}
+export SOURCE_CHECK_ONLY=${SOURCE_CHECK_ONLY:-"false"}
+export COMPAT=${COMPAT:-1}
+export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
+export PYTEST_PAR=2
+export PYTEST_SENTRY_ALWAYS_REPORT=1
+export BOLTDIR=lightning-rfc
+# Allow up to 4 concurrent tests when not under valgrind, which might run out of memory.
+if [ "$VALGRIND" = 0 ]; then
+    PYTEST_PAR=4
+fi
+export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"}
+
+mkdir -p dependencies/bin || true
+
+# Download bitcoind and bitcoin-cli 
+if [ ! -f dependencies/bin/bitcoind ]; then
+    wget https://storage.googleapis.com/c-lightning-tests/bitcoin-0.18.1-x86_64-linux-gnu.tar.bz2
+    tar -xjf bitcoin-0.18.1-x86_64-linux-gnu.tar.bz2
+    mv bitcoin-0.18.1/bin/* dependencies/bin
+    rm -rf bitcoin-0.18.1-x86_64-linux-gnu.tar.gz bitcoin-0.18.1
+fi
+
+pyenv global 3.7
+
+# Update pip first, may save us the compilation of binary packages in the next call
+pip3 install --user -U --quiet --progress-bar off \
+     pip \
+     pytest-test-groups==1.0.3
+
+pip3 install --user -U --quiet --progress-bar off \
+     -r requirements.txt \
+     -r contrib/pyln-client/requirements.txt \
+     -r contrib/pyln-proto/requirements.txt \
+     -r contrib/pyln-testing/requirements.txt
+
+pip3 install --user -U --quiet --progress-bar off \
+     pytest-sentry \
+     pytest-rerunfailures
+
+echo "Configuration which is going to be built:"
+echo -en 'travis_fold:start:script.1\\r'
+./configure CC="$CC"
+cat config.vars
+echo -en 'travis_fold:end:script.1\\r'
+
+cat > pytest.ini << EOF
+[pytest]
+addopts=-p no:logging --color=no --reruns=5
+EOF
+
+git clone https://github.com/lightningnetwork/lightning-rfc.git
+
+if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ]
+then
+    export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/
+    export MAKE_HOST="$TARGET_HOST"
+    export BUILD=x86_64-pc-linux-gnu
+    export AR="$TARGET_HOST"-ar
+    export AS="$TARGET_HOST"-as
+    export CC="$TARGET_HOST"-gcc
+    export CXX="$TARGET_HOST"-g++
+    export LD="$TARGET_HOST"-ld
+    export STRIP="$TARGET_HOST"-strip
+    export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static
+
+    wget -q https://zlib.net/zlib-1.2.11.tar.gz \
+    && tar xf zlib-1.2.11.tar.gz \
+    && cd zlib-1.2.11 \
+    && ./configure --prefix="$QEMU_LD_PREFIX" \
+    && make \
+    && sudo make install
+    cd .. && rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11
+
+    wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip \
+    && unzip -q sqlite-src-3260000.zip \
+    && cd sqlite-src-3260000 \
+    && automake --add-missing --force-missing --copy || true \
+    && ./configure --disable-tcl --enable-static --disable-readline --disable-threadsafe --disable-load-extension --host="$TARGET_HOST" --prefix="$QEMU_LD_PREFIX" \
+    && make \
+    && sudo make install
+    cd .. && rm sqlite-src-3260000.zip && rm -rf sqlite-src-3260000
+
+    wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz \
+    && tar xf gmp-6.1.2.tar.xz \
+    && cd gmp-6.1.2 \
+    && ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST" \
+    && make \
+    && sudo make install
+    cd .. && rm gmp-6.1.2.tar.xz && rm -rf gmp-6.1.2
+
+    ./configure --enable-static
+
+    echo -en 'travis_fold:start:script.2\\r'
+    make -j3 > /dev/null
+    echo -en 'travis_fold:end:script.2\\r'
+
+    # Tests would need to be wrapped with qemu-<arch>-static
+    #echo -en 'travis_fold:start:script.3\\r'
+    #make -j$PYTEST_PAR check-units
+    #echo -en 'travis_fold:end:script.3\\r'
+else
+    echo -en 'travis_fold:start:script.2\\r'
+    make -j8 > /dev/null
+    echo -en 'travis_fold:end:script.2\\r'
+
+    echo -en 'travis_fold:start:script.3\\r'
+    $TEST_CMD
+    echo -en 'travis_fold:end:script.3\\r'
+fi

.version

@@ -1 +0,0 @@
-25.05

Cargo.toml

@@ -1,13 +0,0 @@
-[profile.release]
-strip = "debuginfo"
-
-[workspace]
-resolver = "2"
-members = [
-    "cln-rpc",
-    "cln-grpc",
-    "plugins",
-    "plugins/grpc-plugin",
-    "plugins/rest-plugin",
-    "plugins/lsps-plugin",
-]

Dockerfile

@@ -1,35 +1,11 @@
-# This Dockerfile is used by buildx to build ARM64, AMD64, and ARM32 Docker images from an AMD64 host.
-# To speed up the build process, we are cross-compiling rather than relying on QEMU.
-# There are four main stages:
-# * downloader: Downloads specific binaries needed for core lightning for each architecture.
-# * builder: Cross-compiles for each architecture.
-# * builder-python: Builds Python dependencies for wss-proxy with QEMU.
-# * final: Creates the runtime image.
-
-ARG DEFAULT_TARGETPLATFORM="linux/amd64"
-ARG BASE_DISTRO="debian:bookworm-slim"
-
-FROM --platform=$BUILDPLATFORM ${BASE_DISTRO} AS base-downloader
-RUN set -ex \
-	&& apt-get update \
-	&& apt-get install -qq --no-install-recommends ca-certificates dirmngr wget qemu-user-static binfmt-support
-
-FROM base-downloader AS base-downloader-linux-amd64
-ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu
-ENV DESCHASHPLUGIN_ARCH=linux-amd64
-ENV DESCHASHPLUGIN_HASH=deadc00c68fac80b2718d92f69bf06acd8fff646228d497bbb76a4f0a12ca217
-
-FROM base-downloader AS base-downloader-linux-arm64
-ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu
-ENV DESCHASHPLUGIN_ARCH=linux-arm64
-ENV DESCHASHPLUGIN_HASH=d48c3e5aede77bd9cb72d78689ce12c0327f624435cb0496b3eacb92df416363
-
-FROM base-downloader AS base-downloader-linux-arm
-ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf
-ENV DESCHASHPLUGIN_ARCH=linux-arm
-ENV DESCHASHPLUGIN_HASH=f7df336c72dd1674bd18ff23862a410b6a9691a3e13752264dcffa0950e21c74
-
-FROM base-downloader-${TARGETOS}-${TARGETARCH} AS downloader
+# This dockerfile is meant to compile a c-lightning x64 image
+# It is using multi stage build:
+# * downloader: Download litecoin/bitcoin and qemu binaries needed for c-lightning
+# * builder: Compile c-lightning dependencies, then c-lightning itself with static linking
+# * final: Copy the binaries required at runtime
+# The resulting image uploaded to dockerhub will only contain what is needed for runtime.
+# From the root of the repository, run "docker build -t yourimage:yourtag ."
+FROM debian:stretch-slim as downloader
 
 RUN set -ex \
 	&& apt-get update \
@@ -37,308 +13,89 @@ RUN set -ex \
 
 WORKDIR /opt
 
+RUN wget -qO /opt/tini "https://github.com/krallin/tini/releases/download/v0.18.0/tini" \
+    && echo "12d20136605531b09a2c2dac02ccee85e1b874eb322ef6baf7561cd93f93c855 /opt/tini" | sha256sum -c - \
+    && chmod +x /opt/tini
 
-ENV BITCOIN_VERSION=27.1
-ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
+ARG BITCOIN_VERSION=0.18.1
+ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
 ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/$BITCOIN_TARBALL
-ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS
+ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS.asc
 
 RUN mkdir /opt/bitcoin && cd /opt/bitcoin \
     && wget -qO $BITCOIN_TARBALL "$BITCOIN_URL" \
-    && wget -qO bitcoin "$BITCOIN_ASC_URL" \
-    && grep $BITCOIN_TARBALL bitcoin | tee SHA256SUMS \
-    && sha256sum -c SHA256SUMS \
+    && wget -qO bitcoin.asc "$BITCOIN_ASC_URL" \
+    && grep $BITCOIN_TARBALL bitcoin.asc | tee SHA256SUMS.asc \
+    && sha256sum -c SHA256SUMS.asc \
     && BD=bitcoin-$BITCOIN_VERSION/bin \
-    && tar -xzvf $BITCOIN_TARBALL $BD/ --strip-components=1 \
+    && tar -xzvf $BITCOIN_TARBALL $BD/bitcoin-cli --strip-components=1 \
     && rm $BITCOIN_TARBALL
 
 ENV LITECOIN_VERSION 0.16.3
-ENV LITECOIN_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz
+ENV LITECOIN_PGP_KEY FE3348877809386C
+ENV LITECOIN_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-x86_64-linux-gnu.tar.gz
+ENV LITECOIN_ASC_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-linux-signatures.asc
+ENV LITECOIN_SHA256 686d99d1746528648c2c54a1363d046436fd172beadaceea80bdc93043805994
 
 # install litecoin binaries
 RUN mkdir /opt/litecoin && cd /opt/litecoin \
     && wget -qO litecoin.tar.gz "$LITECOIN_URL" \
-    && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \
+    && echo "$LITECOIN_SHA256  litecoin.tar.gz" | sha256sum -c - \
+    && BD=litecoin-$LITECOIN_VERSION/bin \
+    && tar -xzvf litecoin.tar.gz $BD/litecoin-cli --strip-components=1 --exclude=*-qt \
     && rm litecoin.tar.gz
 
-ENV DESCHASHPLUGIN_URL https://github.com/nbd-wtf/invoicewithdescriptionhash/releases/download/v1.4/invoicewithdescriptionhash-v1.4-${DESCHASHPLUGIN_ARCH}.tar.gz
-ENV DESCHASHPLUGIN_SHA256 ${DESCHASHPLUGIN_HASH}
-RUN mkdir /opt/deschashplugin && cd /opt/deschashplugin \
-    && wget -qO invoicewithdescriptionhash.tar.gz "$DESCHASHPLUGIN_URL" \
-    && echo "$DESCHASHPLUGIN_SHA256  invoicewithdescriptionhash.tar.gz" | sha256sum -c - \
-    && tar -xzvf invoicewithdescriptionhash.tar.gz && rm invoicewithdescriptionhash.tar.gz \
-    && chmod a+x invoicewithdescriptionhash
+FROM debian:stretch-slim as builder
 
-FROM --platform=${DEFAULT_TARGETPLATFORM} ${BASE_DISTRO} AS base-builder
-RUN apt-get update -qq && \
-    apt-get install -qq -y --no-install-recommends \
-        autoconf \
-        automake \
-        bison \
-        build-essential \
-        ca-certificates \
-        curl \
-        dirmngr \
-        flex \
-        gettext \
-        git \
-        gnupg \
-        jq \
-        libicu-dev \
-        libtool \
-        libffi-dev \
-        pkg-config \
-        libssl-dev \
-        protobuf-compiler \
-        python3 \
-        python3-dev \
-        python3-mako \
-        python3-pip \
-        python3-venv \
-        python3-setuptools \
-        libev-dev \
-        libevent-dev \
-        qemu-user-static \
-        wget \
-        unzip \
-        tclsh
+ENV LIGHTNINGD_VERSION=master
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates autoconf automake build-essential git libtool python3 python3-mako wget gnupg dirmngr git gettext
 
-ENV PATH="/root/.local/bin:$PATH" \
-    PYTHON_VERSION=3 \
-    POETRY_VERSION=2.0.1
-RUN curl -sSL https://install.python-poetry.org | python3 - && \
-    poetry self add poetry-plugin-export
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install --upgrade pip setuptools wheel
+RUN wget -q https://zlib.net/zlib-1.2.11.tar.gz \
+&& tar xvf zlib-1.2.11.tar.gz \
+&& cd zlib-1.2.11 \
+&& ./configure \
+&& make \
+&& make install && cd .. && rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11
 
-RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz -O zlib.tar.gz && \
-    wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip -O sqlite.zip && \
-    wget -q https://ftp.postgresql.org/pub/source/v17.1/postgresql-17.1.tar.gz -O postgres.tar.gz
+RUN apt-get install -y --no-install-recommends unzip tclsh \
+&& wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip \
+&& unzip sqlite-src-3260000.zip \
+&& cd sqlite-src-3260000 \
+&& ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension \
+&& make \
+&& make install && cd .. && rm sqlite-src-3260000.zip && rm -rf sqlite-src-3260000
+
+RUN wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz \
+&& tar xvf gmp-6.1.2.tar.xz \
+&& cd gmp-6.1.2 \
+&& ./configure --disable-assembly \
+&& make \
+&& make install && cd .. && rm gmp-6.1.2.tar.xz && rm -rf gmp-6.1.2
 
 WORKDIR /opt/lightningd
 COPY . /tmp/lightning
 RUN git clone --recursive /tmp/lightning . && \
     git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD)
 
-# Do not build python plugins (wss-proxy) here, python doesn't support cross compilation.
-RUN sed -i '/^wss-proxy/d' pyproject.toml && \
-    poetry lock && \
-    poetry export -o requirements.txt --without-hashes
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install -r requirements.txt && \
-    pip3 cache purge
-WORKDIR /
+ARG DEVELOPER=0
+RUN ./configure --prefix=/tmp/lightning_install --enable-static && make -j3 DEVELOPER=${DEVELOPER} && make install
 
-FROM base-builder AS base-builder-linux-amd64
+FROM debian:stretch-slim as final
+COPY --from=downloader /opt/tini /usr/bin/tini
+RUN apt-get update && apt-get install -y --no-install-recommends socat inotify-tools python3 python3-pip \
+    && rm -rf /var/lib/apt/lists/*
 
-ENV POSTGRES_CONFIG="--without-readline" \
-    PG_CONFIG=/usr/local/pgsql/bin/pg_config
-
-FROM base-builder AS base-builder-linux-arm64
-ENV target_host=aarch64-linux-gnu \
-    target_host_rust=aarch64-unknown-linux-gnu \
-    target_host_qemu=qemu-aarch64-static
-
-RUN apt-get install -qq -y --no-install-recommends \
-        libc6-arm64-cross \
-        gcc-${target_host} \
-        g++-${target_host}
-
-ENV AR=${target_host}-ar \
-    AS=${target_host}-as \
-    CC=${target_host}-gcc \
-    CXX=${target_host}-g++ \
-    LD=${target_host}-ld \
-    STRIP=${target_host}-strip \
-    QEMU_LD_PREFIX=/usr/${target_host} \
-    HOST=${target_host} \
-    TARGET=${target_host_rust} \
-    RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
-    PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
-
-ENV ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
-    SQLITE_CONFIG="--host=${target_host} --prefix=${QEMU_LD_PREFIX}" \
-    POSTGRES_CONFIG="--without-readline --prefix=${QEMU_LD_PREFIX}" \
-    PG_CONFIG="${QEMU_LD_PREFIX}/bin/pg_config"
-
-FROM base-builder AS base-builder-linux-arm
-
-ENV target_host=arm-linux-gnueabihf \
-    target_host_rust=armv7-unknown-linux-gnueabihf \
-    target_host_qemu=qemu-arm-static
-
-RUN apt-get install -qq -y --no-install-recommends \
-        libc6-armhf-cross \
-        gcc-${target_host} \
-        g++-${target_host}
-
-ENV AR=${target_host}-ar \
-    AS=${target_host}-as \
-    CC=${target_host}-gcc \
-    CXX=${target_host}-g++ \
-    LD=${target_host}-ld \
-    STRIP=${target_host}-strip \
-    QEMU_LD_PREFIX=/usr/${target_host} \
-    HOST=${target_host} \
-    TARGET=${target_host_rust} \
-    RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \
-    PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig"
-
-ENV ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \
-    SQLITE_CONFIG="--host=${target_host} --prefix=${QEMU_LD_PREFIX}" \
-    POSTGRES_CONFIG="--without-readline --prefix=${QEMU_LD_PREFIX}" \
-    PG_CONFIG="${QEMU_LD_PREFIX}/bin/pg_config"
-
-FROM base-builder-${TARGETOS}-${TARGETARCH} AS builder
-
-ENV LIGHTNINGD_VERSION=master
-
-RUN mkdir zlib && tar xvf zlib.tar.gz -C zlib --strip-components=1 \
-    && cd zlib \
-    && ./configure ${ZLIB_CONFIG} \
-    && make \
-    && make install && cd .. && \
-    rm zlib.tar.gz && \
-    rm -rf zlib
-
-RUN unzip sqlite.zip \
-    && cd sqlite-* \
-    && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension ${SQLITE_CONFIG} \
-    && make \
-    && make install && cd .. && rm sqlite.zip && rm -rf sqlite-*
-
-RUN mkdir postgres && tar xvf postgres.tar.gz -C postgres --strip-components=1 \
-    && cd postgres \
-    && ./configure ${POSTGRES_CONFIG} \
-    && cd src/include \
-    && make install \
-    && cd ../interfaces/libpq \
-    && make install \
-    && cd ../../bin/pg_config \
-    && make install \
-    && cd ../../../../ && \
-    rm postgres.tar.gz && \
-    rm -rf postgres && \
-    ldconfig "$(${PG_CONFIG} --libdir)"
-
-# Save libpq to a specific location to copy it into the final image.
-RUN mkdir /var/libpq && cp -a "$(${PG_CONFIG} --libdir)"/libpq.* /var/libpq
-
-ENV RUST_PROFILE=release \
-    PATH="/root/.cargo/bin:/root/.local/bin:$PATH"
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
-RUN rustup toolchain install stable --component rustfmt --allow-downgrade
-
-COPY --from=downloader /usr/bin/${target_host_qemu} /usr/bin/${target_host_qemu}
-WORKDIR /opt/lightningd
-
-# If cross-compiling, need to tell it to cargo.
-RUN ( ! [ -n "${target_host}" ] ) || \
-    (mkdir -p .cargo && echo "[target.${target_host_rust}]\nlinker = \"${target_host}-gcc\"" > .cargo/config)
-
-# Weird errors with cargo for cln-grpc on arm7 https://github.com/ElementsProject/lightning/issues/6596
-RUN ( ! [ "${target_host}" = "arm-linux-gnueabihf" ] ) || \
-    (sed -i '/documentation = "https:\/\/docs.rs\/cln-grpc"/a include = ["**\/*.*"]' cln-grpc/Cargo.toml)
-
-# Ensure that the desired grpcio-tools & protobuf versions are installed
-# https://github.com/ElementsProject/lightning/pull/7376#issuecomment-2161102381
-RUN poetry lock && poetry install && \
-    poetry self add poetry-plugin-export
-
-# Ensure that git differences are removed before making bineries, to avoid `-modded` suffix
-# poetry.lock changed due to pyln-client, pyln-proto and pyln-testing version updates
-# pyproject.toml was updated to exclude wss-proxy plugins in base-builder stage
-RUN git reset --hard HEAD
-
-RUN ./configure --prefix=/tmp/lightning_install --enable-static && poetry run make install
-
-# Export the requirements for the plugins so we can install them in builder-python stage
-WORKDIR /opt/lightningd/plugins/wss-proxy
-RUN poetry lock && poetry export -o requirements.txt --without-hashes
-WORKDIR /opt/lightningd
-RUN echo 'RUSTUP_INSTALL_OPTS="${RUSTUP_INSTALL_OPTS}"' > /tmp/rustup_install_opts.txt
-
-# We need to build python plugins on the target's arch because python doesn't support cross build
-FROM ${BASE_DISTRO} AS builder-python
-RUN apt-get update -qq && \
-    apt-get install -qq -y --no-install-recommends \
-        git \
-        curl \
-        libtool \
-        pkg-config \
-        autoconf \
-        automake \
-        build-essential \
-        libffi-dev \
-        libssl-dev \
-        python3 \
-        python3-dev \
-        python3-pip \
-        python3-venv && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-ENV PYTHON_VERSION=3
-RUN mkdir -p /root/.venvs && \
-    python3 -m venv /root/.venvs/cln && \
-    . /root/.venvs/cln/bin/activate && \
-    pip3 install --upgrade pip setuptools wheel
-
-# Copy rustup_install_opts.txt file from builder
-COPY --from=builder /tmp/rustup_install_opts.txt /tmp/rustup_install_opts.txt
-# Setup ENV $RUSTUP_INSTALL_OPTS for this stage
-RUN export $(cat /tmp/rustup_install_opts.txt)
-ENV PATH="/root/.cargo/bin:/root/.venvs/cln/bin:$PATH"
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS}
-
-WORKDIR /opt/lightningd/plugins/wss-proxy
-COPY --from=builder /opt/lightningd/plugins/wss-proxy/requirements.txt .
-RUN pip3 install -r requirements.txt
-RUN pip3 cache purge
-
-WORKDIR /opt/lightningd
-
-FROM ${BASE_DISTRO} AS final
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-      tini \
-      socat \
-      inotify-tools \
-      jq \
-      python3 \
-      python3-pip && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-ENV LIGHTNINGD_DATA=/root/.lightning \
-    LIGHTNINGD_RPC_PORT=9835 \
-    LIGHTNINGD_PORT=9735 \
-    LIGHTNINGD_NETWORK=bitcoin
+ENV LIGHTNINGD_DATA=/root/.lightning
+ENV LIGHTNINGD_RPC_PORT=9835
+ENV LIGHTNINGD_PORT=9735
+ENV LIGHTNINGD_NETWORK=bitcoin
 
 RUN mkdir $LIGHTNINGD_DATA && \
-    mkdir /etc/bundledplugins && \
-    mkdir $LIGHTNINGD_DATA/plugins && \
     touch $LIGHTNINGD_DATA/config
 VOLUME [ "/root/.lightning" ]
-
-# Take libpq directly from builder.
-RUN mkdir /var/libpq && mkdir -p /usr/local/pgsql/lib
-RUN --mount=type=bind,from=builder,source=/var/libpq,target=/var/libpq,rw \
-    cp -a /var/libpq/libpq.* /usr/local/pgsql/lib && \
-    echo "/usr/local/pgsql/lib" > /etc/ld.so.conf.d/libpq.conf && \
-    ldconfig
-
 COPY --from=builder /tmp/lightning_install/ /usr/local/
-COPY --from=builder-python /root/.venvs/cln/lib/python3.11/site-packages /usr/local/lib/python3.11/dist-packages/
 COPY --from=downloader /opt/bitcoin/bin /usr/bin
 COPY --from=downloader /opt/litecoin/bin /usr/bin
-COPY --from=downloader /opt/deschashplugin $LIGHTNINGD_DATA/plugins
-COPY --from=downloader /opt/deschashplugin /etc/bundledplugins
 COPY tools/docker-entrypoint.sh entrypoint.sh
 
 EXPOSE 9735 9835

LICENSE

@@ -1,7 +1,7 @@
 Note: the modules in the ccan/ directory have their own licenses, but
 the rest of the code is covered by the following (BSD-MIT) license:
 
-	Copyright Rusty Russell (Blockstream) 2015-2024.
+	Copyright Rusty Russell (Blockstream) 2015.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,10 +1,11 @@
 #! /usr/bin/make
 
 # Extract version from git, or if we're from a zipfile, use dirname
-VERSION=$(shell git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|v\1|gp')
+VERSION=$(shell git describe --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc]*\)$$|\1|gp')
 
-# Next release.
-CLN_NEXT_VERSION := v25.05
+ifeq ($(VERSION),)
+$(error "ERROR: git is required for generating version information")
+endif
 
 # --quiet / -s means quiet, dammit!
 ifeq ($(findstring s,$(word 1, $(MAKEFLAGS))),s)
@@ -16,19 +17,14 @@ SUPPRESS_OUTPUT :=
 endif
 
 DISTRO=$(shell lsb_release -is 2>/dev/null || echo unknown)-$(shell lsb_release -rs 2>/dev/null || echo unknown)
-OS=$(shell uname -s)
-ARCH=$(shell uname -m)
-# Changing this could break installs!
 PKGNAME = c-lightning
 
 # We use our own internal ccan copy.
 CCANDIR := ccan
 
 # Where we keep the BOLT RFCs
-BOLTDIR := ../bolts/
-DEFAULT_BOLTVERSION := ccfa38ed4f592c3711156bb4ded77f44ec01101d
-# Can be overridden on cmdline.
-BOLTVERSION := $(DEFAULT_BOLTVERSION)
+BOLTDIR := ../lightning-rfc/
+BOLTVERSION := 9e8e29af9b9a922eb114b2c716205d0772946e56
 
 -include config.vars
 
@@ -36,7 +32,7 @@ SORT=LC_ALL=C sort
 
 
 ifeq ($V,1)
-VERBOSE = $(ECHO) '$(subst ','\'',$(2))'; $(2)
+VERBOSE = $(ECHO) $(2); $(2)
 else
 VERBOSE = $(ECHO) $(1); $(2)
 endif
@@ -46,7 +42,13 @@ VG=VALGRIND=1 valgrind -q --error-exitcode=7
 VG_TEST_ARGS = --track-origins=yes --leak-check=full --show-reachable=yes --errors-for-leak-kinds=all
 endif
 
-ifeq ($(DEBUGBUILD),1)
+ifneq ($(ASAN),0)
+SANITIZER_FLAGS=-fsanitize=address
+else
+SANITIZER_FLAGS=
+endif
+
+ifeq ($(DEVELOPER),1)
 DEV_CFLAGS=-DCCAN_TAKE_DEBUG=1 -DCCAN_TAL_DEBUG=1 -DCCAN_JSON_OUT_DEBUG=1
 else
 DEV_CFLAGS=
@@ -56,10 +58,6 @@ ifeq ($(COVERAGE),1)
 COVFLAGS = --coverage
 endif
 
-ifeq ($(CLANG_COVERAGE),1)
-COVFLAGS+=-fprofile-instr-generate -fcoverage-mapping
-endif
-
 ifeq ($(PIE),1)
 PIE_CFLAGS=-fPIE -fPIC
 PIE_LDFLAGS=-pie
@@ -67,29 +65,22 @@ endif
 
 ifeq ($(COMPAT),1)
 # We support compatibility with pre-0.6.
-COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1 -DCOMPAT_V061=1 -DCOMPAT_V062=1 -DCOMPAT_V070=1 -DCOMPAT_V072=1 -DCOMPAT_V073=1 -DCOMPAT_V080=1 -DCOMPAT_V081=1 -DCOMPAT_V082=1 -DCOMPAT_V090=1 -DCOMPAT_V0100=1 -DCOMPAT_V0121=1
+COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1 -DCOMPAT_V061=1 -DCOMPAT_V062=1 -DCOMPAT_V070=1 -DCOMPAT_V072=1 -DCOMPAT_V073=1 -DCOMPAT_V080=1 -DCOMPAT_V081=1 -DCOMPAT_V082=1 -DCOMPAT_V090=1
 endif
 
+# Timeout shortly before the 600 second travis silence timeout
 # (method=thread to support xdist)
-PYTEST_OPTS := -v -p no:logging $(PYTEST_OPTS)
-MY_CHECK_PYTHONPATH=$${PYTHONPATH}$${PYTHONPATH:+:}$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7:$(shell pwd)/contrib/pyln-grpc-proto
-# Collect generated python files to be excluded from lint checks
-PYTHON_GENERATED= \
-	contrib/pyln-grpc-proto/pyln/grpc/primitives_pb2.py \
-	contrib/pyln-grpc-proto/pyln/grpc/node_pb2_grpc.py \
-	contrib/pyln-grpc-proto/pyln/grpc/node_pb2.py \
-	contrib/pyln-testing/pyln/testing/grpc2py.py
+PYTEST_OPTS := -v --timeout=550 --timeout_method=thread -p no:logging
 
 # This is where we add new features as bitcoin adds them.
 FEATURES :=
 
 CCAN_OBJS :=					\
 	ccan-asort.o				\
-	ccan-base64.o				\
+	ccan-autodata.o				\
 	ccan-bitmap.o				\
 	ccan-bitops.o				\
 	ccan-breakpoint.o			\
-	ccan-closefrom.o			\
 	ccan-crc32c.o				\
 	ccan-crypto-hmac.o			\
 	ccan-crypto-hkdf.o			\
@@ -121,13 +112,10 @@ CCAN_OBJS :=					\
 	ccan-ptr_valid.o			\
 	ccan-rbuf.o				\
 	ccan-read_write_all.o			\
-	ccan-rune-coding.o			\
-	ccan-rune-rune.o			\
 	ccan-str-base32.o			\
 	ccan-str-hex.o				\
 	ccan-str.o				\
 	ccan-strmap.o				\
-	ccan-strset.o				\
 	ccan-take.o				\
 	ccan-tal-grab_file.o			\
 	ccan-tal-link.o				\
@@ -143,7 +131,7 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/alignof/alignof.h		\
 	$(CCANDIR)/ccan/array_size/array_size.h		\
 	$(CCANDIR)/ccan/asort/asort.h			\
-	$(CCANDIR)/ccan/base64/base64.h			\
+	$(CCANDIR)/ccan/autodata/autodata.h		\
 	$(CCANDIR)/ccan/bitmap/bitmap.h			\
 	$(CCANDIR)/ccan/bitops/bitops.h			\
 	$(CCANDIR)/ccan/breakpoint/breakpoint.h		\
@@ -151,7 +139,6 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/cast/cast.h			\
 	$(CCANDIR)/ccan/cdump/cdump.h			\
 	$(CCANDIR)/ccan/check_type/check_type.h		\
-	$(CCANDIR)/ccan/closefrom/closefrom.h		\
 	$(CCANDIR)/ccan/compiler/compiler.h		\
 	$(CCANDIR)/ccan/container_of/container_of.h	\
 	$(CCANDIR)/ccan/cppmagic/cppmagic.h		\
@@ -165,7 +152,6 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/endian/endian.h			\
 	$(CCANDIR)/ccan/err/err.h			\
 	$(CCANDIR)/ccan/fdpass/fdpass.h			\
-	$(CCANDIR)/ccan/graphql/graphql.h		\
 	$(CCANDIR)/ccan/htable/htable.h			\
 	$(CCANDIR)/ccan/htable/htable_type.h		\
 	$(CCANDIR)/ccan/ilog/ilog.h			\
@@ -180,7 +166,6 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/json_out/json_out.h		\
 	$(CCANDIR)/ccan/likely/likely.h			\
 	$(CCANDIR)/ccan/list/list.h			\
-	$(CCANDIR)/ccan/lqueue/lqueue.h			\
 	$(CCANDIR)/ccan/mem/mem.h			\
 	$(CCANDIR)/ccan/membuf/membuf.h			\
 	$(CCANDIR)/ccan/noerr/noerr.h			\
@@ -192,15 +177,12 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/ptrint/ptrint.h			\
 	$(CCANDIR)/ccan/rbuf/rbuf.h			\
 	$(CCANDIR)/ccan/read_write_all/read_write_all.h	\
-	$(CCANDIR)/ccan/rune/internal.h			\
-	$(CCANDIR)/ccan/rune/rune.h			\
 	$(CCANDIR)/ccan/short_types/short_types.h	\
 	$(CCANDIR)/ccan/str/base32/base32.h		\
 	$(CCANDIR)/ccan/str/hex/hex.h			\
 	$(CCANDIR)/ccan/str/str.h			\
 	$(CCANDIR)/ccan/str/str_debug.h			\
 	$(CCANDIR)/ccan/strmap/strmap.h			\
-	$(CCANDIR)/ccan/strset/strset.h			\
 	$(CCANDIR)/ccan/structeq/structeq.h		\
 	$(CCANDIR)/ccan/take/take.h			\
 	$(CCANDIR)/ccan/tal/grab_file/grab_file.h	\
@@ -218,53 +200,12 @@ CDUMP_OBJS := ccan-cdump.o ccan-strmap.o
 
 BOLT_GEN := tools/generate-wire.py
 WIRE_GEN := $(BOLT_GEN)
+BOLT_DEPS := $(BOLT_GEN)
 
-# If you use wiregen, you're dependent on the tool and its templates
-WIRE_GEN_DEPS := $(WIRE_GEN) $(wildcard tools/gen/*_template)
-
-# These are filled by individual Makefiles
-ALL_PROGRAMS :=
-ALL_TEST_PROGRAMS :=
-ALL_TEST_GEN :=
-ALL_FUZZ_TARGETS :=
-ALL_C_SOURCES :=
-ALL_C_HEADERS :=
-# Extra (non C) targets that should be built by default.
-DEFAULT_TARGETS :=
-
-# Installation directories
-exec_prefix = $(PREFIX)
-bindir = $(exec_prefix)/bin
-libexecdir = $(exec_prefix)/libexec
-pkglibexecdir = $(libexecdir)/$(PKGNAME)
-plugindir = $(pkglibexecdir)/plugins
-datadir = $(PREFIX)/share
-docdir = $(datadir)/doc/$(PKGNAME)
-mandir = $(datadir)/man
-man1dir = $(mandir)/man1
-man5dir = $(mandir)/man5
-man7dir = $(mandir)/man7
-man8dir = $(mandir)/man8
-
-# M1 macos machines with homebrew will install the native libraries in
-# /opt/homebrew instead of /usr/local, most likely because they
-# emulate x86_64 compatibility via Rosetta, and wanting to keep the
-# libraries separate. This however means we also need to switch out
-# the paths accordingly when we detect we're on an M1 macos machine.
-ifeq ("$(OS)-$(ARCH)", "Darwin-arm64")
-CPATH := /opt/homebrew/include
-LIBRARY_PATH := /opt/homebrew/lib
-LDFLAGS := -L/opt/homebrew/opt/sqlite/lib
-CPPFLAGS := -I/opt/homebrew/opt/sqlite/include
-PKG_CONFIG_PATH=/opt/homebrew/opt/sqlite/lib/pkgconfig
-else
-CPATH := /usr/local/include
-LIBRARY_PATH := /usr/local/lib
-endif
-
-CPPFLAGS += -DCLN_NEXT_VERSION="\"$(CLN_NEXT_VERSION)\"" -DPKGLIBEXECDIR="\"$(pkglibexecdir)\"" -DBINDIR="\"$(bindir)\"" -DPLUGINDIR="\"$(plugindir)\"" -DCCAN_TAL_NEVER_RETURN_NULL=1
-CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I$(CPATH) $(SQLITE3_CFLAGS) $(SODIUM_CFLAGS) $(POSTGRES_INCLUDE) $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) $(CSANFLAGS)
+ALL_PROGRAMS =
 
+CPPFLAGS += -DBINTOPKGLIBEXECDIR="\"$(shell sh tools/rel.sh $(bindir) $(pkglibexecdir))\""
+CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I/usr/local/include $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) -DBUILD_ELEMENTS=1
 # If CFLAGS is already set in the environment of make (to whatever value, it
 # does not matter) then it would export it to subprocesses with the above value
 # we set, including CWARNFLAGS which by default contains -Wall -Werror. This
@@ -275,87 +216,41 @@ unexport CFLAGS
 # We can get configurator to run a different compile cmd to cross-configure.
 CONFIGURATOR_CC := $(CC)
 
-LDFLAGS += $(PIE_LDFLAGS) $(CSANFLAGS) $(COPTFLAGS)
-
+LDFLAGS += $(PIE_LDFLAGS) $(SANITIZER_FLAGS) $(COPTFLAGS)
 ifeq ($(STATIC),1)
 # For MacOS, Jacob Rapoport <jacob@rumblemonkey.com> changed this to:
-#  -L/usr/local/lib -lsqlite3 -lz -Wl,-lm -lpthread -ldl $(COVFLAGS)
+#  -L/usr/local/lib -Wl,-lgmp -lsqlite3 -lz -Wl,-lm -lpthread -ldl $(COVFLAGS)
 # But that doesn't static link.
-LDLIBS = -L$(CPATH) -Wl,-dn $(SQLITE3_LDLIBS) -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
+LDLIBS = -L/usr/local/lib -Wl,-dn -lgmp -lsqlite3 -lz -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
 else
-LDLIBS = -L$(CPATH) -lm $(SQLITE3_LDLIBS) $(COVFLAGS)
+LDLIBS = -L/usr/local/lib -lm -lgmp -lsqlite3 -lz $(COVFLAGS)
 endif
 
 # If we have the postgres client library we need to link against it as well
 ifeq ($(HAVE_POSTGRES),1)
-LDLIBS += $(POSTGRES_LDLIBS)
+LDLIBS += -lpq
 endif
 
-default: show-flags gen all-programs all-test-programs doc-all default-targets $(PYTHON_GENERATED)
+default: show-flags all-programs all-test-programs doc-all
 
-ifneq ($(SUPPRESS_GENERATION),1)
-FORCE = FORCE
-FORCE:
-endif
-
-show-flags: config.vars
+show-flags:
 	@$(ECHO) "CC: $(CC) $(CFLAGS) -c -o"
 	@$(ECHO) "LD: $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) -o"
 
-# We will re-generate, but we won't generate for the first time!
-ccan/config.h config.vars &: configure ccan/tools/configurator/configurator.c
-	@if [ ! -f config.vars ]; then echo 'File config.vars not found: you must run ./configure before running make.' >&2; exit 1; fi
+ccan/config.h: config.vars configure ccan/tools/configurator/configurator.c
 	./configure --reconfigure
 
+config.vars:
+	@echo 'File config.vars not found: you must run ./configure before running make.' >&2
+	@exit 1
+
 %.o: %.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 
-# tools/update-mocks.sh does nasty recursive make, must not do this!
-ifeq ($(SUPPRESS_GENERATION),1)
-SHA256STAMP_CHANGED = false
-SHA256STAMP = exit 1
-else
-# Git doesn't maintain timestamps, so we only regen if sources actually changed:
-# We place the SHA inside some generated files so we can tell if they need updating.
-# Usage: $(call SHA256STAMP_CHANGED)
-SHA256STAMP_CHANGED = [ x"`sed -n 's/.*SHA256STAMP:\([a-f0-9]*\).*/\1/p' $@ 2>/dev/null`" != x"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`" ]
-# Usage: $(call SHA256STAMP,commentprefix,commentpostfix)
-SHA256STAMP = echo "$(1) SHA256STAMP:"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`"$(2)" >> $@
-endif
-
-# generate-wire.py --page [header|impl] hdrfilename wirename < csv > file
-%_wiregen.h: %_wire.csv $(WIRE_GEN_DEPS)
-	@if $(call SHA256STAMP_CHANGED); then \
-		$(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \
-	fi
-
-%_wiregen.c: %_wire.csv $(WIRE_GEN_DEPS)
-	@if $(call SHA256STAMP_CHANGED); then \
-		$(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \
-	fi
-
-%_printgen.h: %_wire.csv $(WIRE_GEN_DEPS)
-	@if $(call SHA256STAMP_CHANGED); then \
-		$(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \
-	fi
-
-%_printgen.c: %_wire.csv $(WIRE_GEN_DEPS)
-	@if $(call SHA256STAMP_CHANGED); then \
-		$(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \
-	fi
-
-RUST_PROFILE ?= debug
-ifneq ($(RUST_PROFILE),debug)
-CARGO_OPTS := --profile=$(RUST_PROFILE) --quiet
-else
-CARGO_OPTS := --quiet
-endif
-
 include external/Makefile
 include bitcoin/Makefile
 include common/Makefile
 include wire/Makefile
-include db/Makefile
 include hsmd/Makefile
 include gossipd/Makefile
 include openingd/Makefile
@@ -366,91 +261,22 @@ include connectd/Makefile
 include lightningd/Makefile
 include cli/Makefile
 include doc/Makefile
-include contrib/msggen/Makefile
 include devtools/Makefile
 include tools/Makefile
-ifneq ($(RUST),0)
-include cln-rpc/Makefile
-include cln-grpc/Makefile
-endif
 include plugins/Makefile
 include tests/plugins/Makefile
 
-ifneq ($(FUZZING),0)
-	include tests/fuzz/Makefile
-endif
+# Generated from PLUGINS definition in plugins/Makefile
+gen_list_of_builtin_plugins.h : plugins/Makefile Makefile
+	@echo GEN $@
+	@rm -f $@ || true
+	@echo 'static const char *list_of_builtin_plugins[] = {' >> $@
+	@echo '$(PLUGINS)' | sed 's@plugins/\([^ 	]*\)@"\1",@g'>> $@
+	@echo 'NULL' >> $@
+	@echo '};' >> $@
 
-ifneq ($V,1)
-MSGGEN_ARGS := -s
-endif
-
-$(MSGGEN_GENALL)&: contrib/msggen/msggen/schema.json
-	@$(call VERBOSE, "msggen $@", PYTHONPATH=contrib/msggen $(PYTHON) contrib/msggen/msggen/__main__.py $(MSGGEN_ARGS) generate)
-
-# The compiler assumes that the proto files are in the same
-# directory structure as the generated files will be. Since we
-# don't do that we need to path the files up.
-GRPC_DIR = contrib/pyln-grpc-proto/pyln
-GRPC_PATH = $(GRPC_DIR)/grpc
-
-GRPC_GEN = \
-	$(GRPC_PATH)/node_pb2.py \
-	$(GRPC_PATH)/node_pb2_grpc.py \
-	$(GRPC_PATH)/primitives_pb2.py
-
-ALL_TEST_GEN += $(GRPC_GEN)
-
-$(GRPC_GEN) &: cln-grpc/proto/node.proto cln-grpc/proto/primitives.proto
-	$(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/node.proto --python_out=$(GRPC_PATH)/ --grpc_python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
-	$(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/primitives.proto --python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional
-	find $(GRPC_DIR)/ -type f -name "*.py" -print0 | xargs -0 sed -i'.bak' -e 's/^import \(.*\)_pb2 as .*__pb2/from pyln.grpc import \1_pb2 as \1__pb2/g'
-	find $(GRPC_DIR)/ -type f -name "*.py.bak" -print0 | xargs -0 rm -f
-
-# We make pretty much everything depend on these.
-ALL_GEN_HEADERS := $(filter %gen.h,$(ALL_C_HEADERS))
-ALL_GEN_SOURCES := $(filter %gen.c,$(ALL_C_SOURCES))
-ALL_NONGEN_HEADERS := $(filter-out %gen.h,$(ALL_C_HEADERS))
-ALL_NONGEN_SOURCES := $(filter-out %gen.c,$(ALL_C_SOURCES))
-ALL_NONGEN_SRCFILES := $(ALL_NONGEN_HEADERS) $(ALL_NONGEN_SOURCES)
-
-# Programs to install in bindir and pkglibexecdir.
-# TODO: $(EXEEXT) support for Windows?  Needs more coding for
-# the individual Makefiles, however.
-BIN_PROGRAMS = \
-	       cli/lightning-cli \
-	       lightningd/lightningd \
-	       tools/lightning-hsmtool\
-	       tools/reckless
-PKGLIBEXEC_PROGRAMS = \
-	       lightningd/lightning_channeld \
-	       lightningd/lightning_closingd \
-	       lightningd/lightning_connectd \
-	       lightningd/lightning_dualopend \
-	       lightningd/lightning_gossipd \
-	       lightningd/lightning_hsmd \
-	       lightningd/lightning_onchaind \
-	       lightningd/lightning_openingd \
-	       lightningd/lightning_websocketd
-
-mkdocs.yml: $(MANPAGES:=.md)
-	@$(call VERBOSE, "genidx $@", \
-	  find doc -maxdepth 1 -name '*\.[0-9]\.md' | \
-	  cut -b 5- | LC_ALL=C sort | \
-	  sed 's/\(.*\)\.\(.*\).*\.md/- "\1": "\1.\2.md"/' | \
-	  $(PYTHON) devtools/blockreplace.py mkdocs.yml manpages --language=yml --indent "          " \
-	)
-
-
-
-# Don't delete these intermediaries.
-.PRECIOUS: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) $(PYTHON_GENERATED)
-
-# Every single object file.
-ALL_OBJS := $(ALL_C_SOURCES:.c=.o)
-
-# We always regen wiregen and printgen files, since SHA256STAMP protects against
-# spurious rebuilds.
-$(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(ALL_C_HEADERS) $(ALL_C_SOURCES)): $(FORCE)
+# Git doesn't maintain timestamps, so we only regen if git says we should.
+CHANGED_FROM_GIT = [ x"`git log $@ | head -n1`" != x"`git log $< | head -n1`" -o x"`git diff $<`" != x"" ]
 
 ifneq ($(TEST_GROUP_COUNT),)
 PYTEST_OPTS += --test-group=$(TEST_GROUP) --test-group-count=$(TEST_GROUP_COUNT)
@@ -466,29 +292,17 @@ else
 PYTEST_OPTS += -x
 endif
 
-# Allow for targeting specific tests by setting the PYTEST_TESTS environment variable.
-ifeq ($(PYTEST_TESTS),)
-PYTEST_TESTS = "tests/"
-endif
-
 check-units:
 
 check: check-units installcheck pytest
 
-pytest: $(ALL_PROGRAMS) $(DEFAULT_TARGETS) $(ALL_TEST_PROGRAMS) $(ALL_TEST_GEN)
+pytest: $(ALL_PROGRAMS)
 ifeq ($(PYTEST),)
 	@echo "py.test is required to run the integration tests, please install using 'pip3 install -r requirements.txt', and rerun 'configure'."
 	exit 1
 else
-# Explicitly hand VALGRIND so you can override on make cmd line.
-	PYTHONPATH=$(MY_CHECK_PYTHONPATH) TEST_DEBUG=1 VALGRIND=$(VALGRIND) $(PYTEST) $(PYTEST_TESTS) $(PYTEST_OPTS)
-endif
-
-check-fuzz: $(ALL_FUZZ_TARGETS)
-ifneq ($(FUZZING),0)
-	@tests/fuzz/check-fuzz.sh
-else
-	@echo "fuzzing is not enabled: first run './configure --enable-fuzzing'"
+# Explicitly hand DEVELOPER and VALGRIND so you can override on make cmd line.
+	PYTHONPATH=`pwd`/contrib/pyln-client:`pwd`/contrib/pyln-testing:`pwd`/contrib/pyln-proto/:$(PYTHONPATH) TEST_DEBUG=1 DEVELOPER=$(DEVELOPER) VALGRIND=$(VALGRIND) $(PYTEST) tests/ $(PYTEST_OPTS)
 endif
 
 # Keep includes in alpha order.
@@ -504,17 +318,11 @@ check-hdr-include-order/%: %
 check-makefile:
 	@if [ x"$(CCANDIR)/config.h `find $(CCANDIR)/ccan -name '*.h' | grep -v /test/ | $(SORT) | tr '\n' ' '`" != x"$(CCAN_HEADERS) " ]; then echo CCAN_HEADERS incorrect; exit 1; fi
 
-# We exclude test files, which need to do weird include tricks!
-SRC_TO_CHECK := $(filter-out $(ALL_TEST_PROGRAMS:=.c), $(ALL_NONGEN_SOURCES))
-check-src-includes: $(SRC_TO_CHECK:%=check-src-include-order/%)
-check-hdr-includes: $(ALL_NONGEN_HEADERS:%=check-hdr-include-order/%)
-
-# If you want to check a specific variant of quotes use:
-#   make check-source-bolt BOLTVERSION=xxx
-ifeq ($(BOLTVERSION),$(DEFAULT_BOLTVERSION))
-CHECK_BOLT_PREFIX=
-else
+# Experimental quotes quote the exact version.
+ifeq ($(EXPERIMENTAL_FEATURES),1)
 CHECK_BOLT_PREFIX=--prefix="BOLT-$(BOLTVERSION)"
+else
+CHECK_BOLT_PREFIX=
 endif
 
 # Any mention of BOLT# must be followed by an exact quote, modulo whitespace.
@@ -526,41 +334,42 @@ LOCAL_BOLTDIR=.tmp.lightningrfc
 bolt-precheck:
 	@[ -d $(BOLTDIR) ] || exit 0; set -e; if [ -z "$(BOLTVERSION)" ]; then rm -rf $(LOCAL_BOLTDIR); ln -sf $(BOLTDIR) $(LOCAL_BOLTDIR); exit 0; fi; [ "$$(git -C $(LOCAL_BOLTDIR) rev-list --max-count=1 HEAD 2>/dev/null)" != "$(BOLTVERSION)" ] || exit 0; rm -rf $(LOCAL_BOLTDIR) && git clone -q $(BOLTDIR) $(LOCAL_BOLTDIR) && cd $(LOCAL_BOLTDIR) && git checkout -q $(BOLTVERSION)
 
-check-source-bolt: $(ALL_NONGEN_SRCFILES:%=bolt-check/%)
+check-source-bolt: $(ALL_TEST_PROGRAMS:%=bolt-check/%.c)
 
 check-whitespace/%: %
 	@if grep -Hn '[ 	]$$' $<; then echo Extraneous whitespace found >&2; exit 1; fi
 
-check-whitespace: check-whitespace/Makefile check-whitespace/tools/check-bolt.c $(ALL_NONGEN_SRCFILES:%=check-whitespace/%)
+check-whitespace: check-whitespace/Makefile check-whitespace/tools/check-bolt.c $(ALL_TEST_PROGRAMS:%=check-whitespace/%.c)
+
+check-markdown:
+	@tools/check-markdown.sh
 
 check-spelling:
 	@tools/check-spelling.sh
 
-PYSRC=$(shell git ls-files "*.py" | grep -v /text.py)
+PYSRC=$(shell git ls-files "*.py" | grep -v /text.py) contrib/pylightning/lightning-pay
 
-# Some tests in pyln will need to find lightningd to run, so have a PATH that
-# allows it to find that
-PYLN_PATH=$(shell pwd)/lightningd:$(PATH)
-check-pyln-%: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS)
-	@(cd contrib/$(shell echo $@ | cut -b 7-) && PATH=$(PYLN_PATH) PYTHONPATH=$(MY_CHECK_PYTHONPATH) $(MAKE) check)
-
-check-python: check-python-flake8 check-pytest-pyln-proto check-pyln-client check-pyln-testing
-
-check-python-flake8:
+check-python:
 	@# E501 line too long (N > 79 characters)
 	@# E731 do not assign a lambda expression, use a def
 	@# W503: line break before binary operator
-	@# E741: ambiguous variable name
-	@flake8 --ignore=E501,E731,E741,W503,F541,E275 --exclude $(shell echo ${PYTHON_GENERATED} | sed 's/ \+/,/g') ${PYSRC}
+	@flake8 --ignore=E501,E731,W503 ${PYSRC}
 
-check-pytest-pyln-proto:
-	PATH=$(PYLN_PATH) PYTHONPATH=$(MY_CHECK_PYTHONPATH) $(PYTEST) contrib/pyln-proto/tests/
+	PYTHONPATH=contrib/pyln-client:$$PYTHONPATH $(PYTEST) contrib/pyln-client/
+	PYTHONPATH=contrib/pyln-proto:$$PYTHONPATH $(PYTEST) contrib/pyln-proto/
 
-check-includes: check-src-includes check-hdr-includes
+check-includes:
 	@tools/check-includes.sh
 
+# cppcheck gets confused by list_for_each(head, i, list): thinks i is uninit.
+.cppcheck-suppress:
+	@git ls-files -- "*.c" "*.h" | grep -vE '^ccan/' | xargs grep -n '_for_each' | sed 's/\([^:]*:.*\):.*/uninitvar:\1/' > $@
+
+check-cppcheck: .cppcheck-suppress
+	@trap 'rm -f .cppcheck-suppress' 0; git ls-files -- "*.c" "*.h" | grep -vE '^ccan/' | xargs cppcheck -q --language=c --std=c11 --error-exitcode=1 --suppressions-list=.cppcheck-suppress --inline-suppr
+
 check-shellcheck:
-	@git ls-files -z -- "*.sh" | xargs -0 shellcheck -f gcc
+	@git ls-files -- "*.sh" | xargs shellcheck
 
 check-setup_locale:
 	@tools/check-setup_locale.sh
@@ -569,64 +378,18 @@ check-tmpctx:
 	@if git grep -n 'tal_free[(]tmpctx)' | grep -Ev '^ccan/|/test/|^common/setup.c:|^common/utils.c:'; then echo "Don't free tmpctx!">&2; exit 1; fi
 
 check-discouraged-functions:
-	@if git grep -E "[^a-z_/](fgets|fputs|gets|scanf|sprintf)\(" -- "*.c" "*.h" ":(exclude)ccan/" ":(exclude)contrib/"; then exit 1; fi
+	@if git grep -E "[^a-z_/](fgets|fputs|gets|scanf|sprintf)\(" -- "*.c" "*.h" ":(exclude)ccan/"; then exit 1; fi
 
 # Don't access amount_msat and amount_sat members directly without a good reason
 # since it risks overflow.
 check-amount-access:
 	@! (git grep -nE "(->|\.)(milli)?satoshis" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" | grep -v '/* Raw:')
-	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" | grep -vE "sizeof.struct amount_(m)?sat."
+	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*"
 
-repeat-doc-examples:
-	@for i in $$(seq 1 $(n)); do \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-		echo "Iteration $$i" >> tests/autogenerate-examples-repeat.log; \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-		VALGRIND=0 TIMEOUT=40 TEST_DEBUG=1 GENERATE_EXAMPLES=1 pytest -vvv tests/autogenerate-rpc-examples.py; \
-		git diff >> tests/autogenerate-examples-repeat.log; \
-		git reset --hard; \
-		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-	done
-
-update-doc-examples:
-	TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 $(PYTEST) $(PYTEST_OPTS) --timeout=1200 tests/autogenerate-rpc-examples.py && $(MAKE) $(MSGGEN_GEN_ALL)
-
-check-doc-examples: update-doc-examples
-	git diff --exit-code HEAD
-
-# For those without working cppcheck
-check-source-no-cppcheck: check-makefile check-source-bolt check-whitespace check-spelling check-python check-includes check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access
-
-check-source: check-source-no-cppcheck
+check-source: check-makefile check-source-bolt check-whitespace check-markdown check-spelling check-python check-includes check-cppcheck check-shellcheck check-setup_locale check-tmpctx check-discouraged-functions check-amount-access
 
 full-check: check check-source
 
-# Simple target to be used on CI systems to check that all the derived
-# files were checked in and updated. It depends on the generated
-# targets, and checks if any of the tracked files changed. If they did
-# then one of the gen-targets caused this change, meaning either the
-# gen-target is not reproducible or the files were forgotten.
-#
-# Do not run on your development tree since it will complain if you
-# have a dirty tree.
-CHECK_GEN_ALL = \
-	$(CLN_GRPC_GENALL) \
-	$(CLN_RPC_GENALL) \
-	$(MANPAGES) \
-	$(WALLET_DB_QUERIES) \
-	$(PYTHON_GENERATED) \
-	$(ALL_GEN_HEADERS) \
-	$(ALL_GEN_SOURCES) \
-	$(MSGGEN_GEN_ALL) \
-	wallet/statements_gettextgen.po \
-	doc/index.rst
-
-gen:  $(CHECK_GEN_ALL)
-
-check-gen-updated:  $(CHECK_GEN_ALL)
-	@echo "Checking for generated files being changed by make"
-	git diff --exit-code HEAD
-
 coverage/coverage.info: check pytest
 	mkdir coverage || true
 	lcov --capture --directory . --output-file coverage/coverage.info
@@ -640,11 +403,9 @@ ncc: ${TARGET_DIR}/libwally-core-build/src/libwallycore.la
 	$(MAKE) CC="ncc -ncgcc -ncld -ncfabs" AR=nccar LD=nccld
 
 # Ignore test/ directories.
-TAGS:
-	$(RM) TAGS; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 etags --append
-
-tags:
-	$(RM) tags; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 ctags --append
+TAGS: FORCE
+	$(RM) TAGS; find * -name test -type d -prune -o -name '*.[ch]' -print -o -name '*.py' -print | xargs etags --append
+FORCE::
 
 ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS)
 
@@ -652,54 +413,38 @@ ALL_PROGRAMS += ccan/ccan/cdump/tools/cdump-enumstr
 # Can't add to ALL_OBJS, as that makes a circular dep.
 ccan/ccan/cdump/tools/cdump-enumstr.o: $(CCAN_HEADERS) Makefile
 
-# Without a working git, you can't generate this file, so assume if it exists
-# it is ok (fixes "sudo make install").
-ifeq ($(VERSION),)
-version_gen.h:
-	echo "ERROR: git is required for generating version information" >&2
-	exit 1
-else
-version_gen.h: $(FORCE)
+gen_version.h: FORCE
 	@(echo "#define VERSION \"$(VERSION)\"" && echo "#define BUILD_FEATURES \"$(FEATURES)\"") > $@.new
 	@if cmp $@.new $@ >/dev/null 2>&1; then rm -f $@.new; else mv $@.new $@; $(ECHO) Version updated; fi
-endif
 
 # That forces this rule to be run every time, too.
-header_versions_gen.h: tools/headerversions $(FORCE)
+gen_header_versions.h: tools/headerversions
 	@tools/headerversions $@
 
-# We make a static library, this way linker can discard unused parts.
-libccan.a: $(CCAN_OBJS)
-	@$(call VERBOSE, "ar $@", $(AR) r $@ $(CCAN_OBJS))
+# Rebuild the world if this changes.
+ALL_GEN_HEADERS += gen_header_versions.h
 
 # All binaries require the external libs, ccan and system library versions.
-$(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): $(EXTERNAL_LIBS) libccan.a
+$(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS): $(EXTERNAL_LIBS) $(CCAN_OBJS)
 
 # Each test program depends on its own object.
-$(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): %: %.o
+$(ALL_TEST_PROGRAMS): %: %.o
 
 # Without this rule, the (built-in) link line contains
 # external/libwallycore.a directly, which causes a symbol clash (it
 # uses some ccan modules internally).  We want to rely on -lwallycore etc.
 # (as per EXTERNAL_LDLIBS) so we filter them out here.
 $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS):
-	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a $($(@)_LDLIBS) -o $@)
-
-# We special case the fuzzing target binaries, as they need to link against libfuzzer,
-# which brings its own main().
-FUZZ_LDFLAGS = -fsanitize=fuzzer
-$(ALL_FUZZ_TARGETS):
-	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a $(FUZZ_LDFLAGS) -o $@)
-
+	@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) -o $@)
 
 # Everything depends on the CCAN headers, and Makefile
-$(CCAN_OBJS) $(CDUMP_OBJS): $(CCAN_HEADERS) Makefile ccan_compat.h
+$(CCAN_OBJS) $(CDUMP_OBJS): $(CCAN_HEADERS) Makefile
 
 # Except for CCAN, we treat everything else as dependent on external/ bitcoin/ common/ wire/ and all generated headers, and Makefile
 $(ALL_OBJS): $(BITCOIN_HEADERS) $(COMMON_HEADERS) $(CCAN_HEADERS) $(WIRE_HEADERS) $(ALL_GEN_HEADERS) $(EXTERNAL_HEADERS) Makefile
 
-# Test files can literally #include generated C files.
-$(ALL_TEST_PROGRAMS:=.o): $(ALL_GEN_SOURCES)
+# We generate headers in two ways, so regen when either changes (or Makefile)
+$(ALL_GEN_HEADERS): ccan/ccan/cdump/tools/cdump-enumstr $(WIRE_GEN) Makefile
 
 update-ccan:
 	mv ccan ccan.old
@@ -714,9 +459,7 @@ update-ccan:
 
 # Now ALL_PROGRAMS is fully populated, we can expand it.
 all-programs: $(ALL_PROGRAMS)
-all-fuzz-programs: $(ALL_FUZZ_TARGETS)
-all-test-programs: $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS)
-default-targets: $(DEFAULT_TARGETS)
+all-test-programs: $(ALL_TEST_PROGRAMS)
 
 distclean: clean
 	$(RM) ccan/config.h config.vars
@@ -724,68 +467,37 @@ distclean: clean
 maintainer-clean: distclean
 	@echo 'This command is intended for maintainers to use; it'
 	@echo 'deletes files that may need special tools to rebuild.'
-	$(RM) $(PYTHON_GENERATED)
 
-# We used to have gen_ files, now we have _gen files.
-# We used to generate doc/schemas/lightning-sql.json.
-obsclean:
-	$(RM) gen_*.h */gen_*.[ch] */*/gen_*.[ch]
-	$(RM) doc/schemas/lightning-sql.json
-
-clean: obsclean
-	$(RM) libccan.a $(CCAN_OBJS) $(CDUMP_OBJS) $(ALL_OBJS)
-	$(RM) $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES)
-	$(RM) $(ALL_PROGRAMS)
-	$(RM) $(ALL_TEST_PROGRAMS)
-	$(RM) $(ALL_FUZZ_TARGETS)
-	$(RM) $(MSGGEN_GEN_ALL)
-	$(RM) ccan/tools/configurator/configurator
+clean:
+	$(RM) $(CCAN_OBJS) $(CDUMP_OBJS) $(ALL_OBJS)
+	$(RM) $(ALL_PROGRAMS) $(ALL_PROGRAMS:=.o)
+	$(RM) $(ALL_TEST_PROGRAMS) $(ALL_TEST_PROGRAMS:=.o)
+	$(RM) gen_*.h ccan/tools/configurator/configurator
 	$(RM) ccan/ccan/cdump/tools/cdump-enumstr.o
 	find . -name '*gcda' -delete
 	find . -name '*gcno' -delete
 	find . -name '*.nccout' -delete
-	if [ "${RUST}" -eq "1" ]; then cargo clean; fi
 
-
-PYLNS=client proto testing
-# See doc/contribute-to-core-lightning/contributor-workflow.md
-update-versions: update-pyln-versions update-wss-proxy-version update-poetry-lock update-dot-version update-doc-examples
-
-update-pyln-versions: $(PYLNS:%=update-pyln-version-%)
-
-update-pyln-version-%:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	cd contrib/pyln-$* && $(MAKE) upgrade-version
-
-pyln-release:  $(PYLNS:%=pyln-release-%)
-
-pyln-release-%:
-	cd contrib/pyln-$* && $(MAKE) prod-release
-
-update-wss-proxy-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	cd plugins/wss-proxy && $(MAKE) upgrade-version
-
-update-poetry-lock:
-	poetry update wss-proxy pyln-client pyln-proto pyln-testing update-reckless-version
-
-update-reckless-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	@sed -i "s/__VERSION__ = '\([.-z]*\)'/__VERSION__ = '$(NEW_VERSION)'/" tools/reckless
-
-update-dot-version:
-	@if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi
-	echo $(NEW_VERSION) > .version
-
-update-mocks: $(ALL_TEST_PROGRAMS:%=update-mocks/%.c)
-
-$(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) libccan.a ccan/ccan/cdump/tools/cdump-enumstr config.vars
-
-update-mocks/%: % $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES)
+update-mocks: $(ALL_GEN_HEADERS)
+update-mocks/%: %
 	@MAKE=$(MAKE) tools/update-mocks.sh "$*" $(SUPPRESS_OUTPUT)
 
-unittest/%: % bolt-precheck
-	BOLTDIR=$(LOCAL_BOLTDIR) $(VG) $(VG_TEST_ARGS) $* > /dev/null
+unittest/%: %
+	$(VG) $(VG_TEST_ARGS) $* > /dev/null
+
+# Installation directories
+exec_prefix = $(PREFIX)
+bindir = $(exec_prefix)/bin
+libexecdir = $(exec_prefix)/libexec
+pkglibexecdir = $(libexecdir)/$(PKGNAME)
+plugindir = $(pkglibexecdir)/plugins
+datadir = $(PREFIX)/share
+docdir = $(datadir)/doc/$(PKGNAME)
+mandir = $(datadir)/man
+man1dir = $(mandir)/man1
+man5dir = $(mandir)/man5
+man7dir = $(mandir)/man7
+man8dir = $(mandir)/man8
 
 # Commands
 MKDIR_P = mkdir -p
@@ -813,21 +525,35 @@ installdirs:
 	$(MKDIR_P) $(DESTDIR)$(man8dir)
 	$(MKDIR_P) $(DESTDIR)$(docdir)
 
+# Programs to install in bindir and pkglibexecdir.
+# TODO: $(EXEEXT) support for Windows?  Needs more coding for
+# the individual Makefiles, however.
+BIN_PROGRAMS = \
+	       cli/lightning-cli \
+	       lightningd/lightningd \
+	       tools/lightning-hsmtool
+PKGLIBEXEC_PROGRAMS = \
+	       lightningd/lightning_channeld \
+	       lightningd/lightning_closingd \
+	       lightningd/lightning_connectd \
+	       lightningd/lightning_gossipd \
+	       lightningd/lightning_hsmd \
+	       lightningd/lightning_onchaind \
+	       lightningd/lightning_openingd
+
 # $(PLUGINS) is defined in plugins/Makefile.
 
-install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS)
+install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS)
 	@$(NORMAL_INSTALL)
 	$(INSTALL_PROGRAM) $(BIN_PROGRAMS) $(DESTDIR)$(bindir)
 	$(INSTALL_PROGRAM) $(PKGLIBEXEC_PROGRAMS) $(DESTDIR)$(pkglibexecdir)
-	@if [ -d "$(DESTDIR)$(plugindir)/clnrest" ]; then rm -rf $(DESTDIR)$(plugindir)/clnrest; fi
 	[ -z "$(PLUGINS)" ] || $(INSTALL_PROGRAM) $(PLUGINS) $(DESTDIR)$(plugindir)
-	for PY in $(PY_PLUGINS); do DIR=`dirname $$PY`; DST=$(DESTDIR)$(plugindir)/`basename $$DIR`; if [ -d $$DST ]; then rm -rf $$DST; fi; $(INSTALL_PROGRAM) -d $$DIR; cp -a $$DIR $$DST ; done
 
 MAN1PAGES = $(filter %.1,$(MANPAGES))
 MAN5PAGES = $(filter %.5,$(MANPAGES))
 MAN7PAGES = $(filter %.7,$(MANPAGES))
 MAN8PAGES = $(filter %.8,$(MANPAGES))
-DOC_DATA = README.md LICENSE
+DOC_DATA = README.md doc/INSTALL.md doc/HACKING.md LICENSE
 
 install-data: installdirs $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA)
 	@$(NORMAL_INSTALL)
@@ -839,27 +565,6 @@ install-data: installdirs $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(
 
 install: install-program install-data
 
-# Non-artifacts that are needed for testing. These are added to the
-# testpack.tar, used to transfer things between builder and tester
-# phase. If you get a missing file/executable while testing on CI it
-# is likely missing from this variable.
-TESTBINS = \
-	$(CLN_PLUGIN_EXAMPLES) \
-	tests/plugins/test_libplugin \
-	tests/plugins/channeld_fakenet \
-	tests/plugins/test_selfdisable_after_getmanifest \
-	tools/hsmtool
-
-# The testpack is used in CI to transfer built artefacts between the
-# build and the test phase. This is necessary because the fixtures in
-# `tests/` explicitly use the binaries built in the current directory
-# rather than using `$PATH`, as that may pick up some other installed
-# version of `lightningd` leading to bogus results. We bundle up all
-# built artefacts here, and will unpack them on the tester (overlaying
-# on top of the checked out repo as if we had just built it in place).
-testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS)
-	tar -caf $@ $^
-
 uninstall:
 	@$(NORMAL_UNINSTALL)
 	@for f in $(BIN_PROGRAMS); do \
@@ -870,10 +575,6 @@ uninstall:
 	  $(ECHO) rm -f $(DESTDIR)$(plugindir)/`basename $$f`; \
 	  rm -f $(DESTDIR)$(plugindir)/`basename $$f`; \
 	done
-	@for f in $(PY_PLUGINS); do \
-	  $(ECHO) rm -rf $(DESTDIR)$(plugindir)/$$(basename $$(dirname $$f)); \
-	  rm -rf $(DESTDIR)$(plugindir)/$$(basename $$(dirname $$f)); \
-	done
 	@for f in $(PKGLIBEXEC_PROGRAMS); do \
 	  $(ECHO) rm -f $(DESTDIR)$(pkglibexecdir)/`basename $$f`; \
 	  rm -f $(DESTDIR)$(pkglibexecdir)/`basename $$f`; \
@@ -910,25 +611,18 @@ installcheck: all-programs
 	fi
 	@rm -rf testinstall || true
 
-version:
-	@echo ${VERSION}
-
 .PHONY: installdirs install-program install-data install uninstall \
-	installcheck ncc bin-tarball show-flags version
+	installcheck ncc bin-tarball show-flags
 
 # Make a tarball of opt/clightning/, optionally with label for distribution.
-ifneq ($(VERSION),)
 bin-tarball: clightning-$(VERSION)-$(DISTRO).tar.xz
 clightning-$(VERSION)-$(DISTRO).tar.xz: DESTDIR=$(shell pwd)/
 clightning-$(VERSION)-$(DISTRO).tar.xz: prefix=opt/clightning
 clightning-$(VERSION)-$(DISTRO).tar.xz: install
 	trap "rm -rf opt" 0; tar cvfa $@ opt/
-endif
 
 ccan-breakpoint.o: $(CCANDIR)/ccan/breakpoint/breakpoint.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
-ccan-base64.o: $(CCANDIR)/ccan/base64/base64.c
-	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-tal.o: $(CCANDIR)/ccan/tal/tal.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-tal-str.o: $(CCANDIR)/ccan/tal/str/str.c
@@ -945,6 +639,8 @@ ccan-list.o: $(CCANDIR)/ccan/list/list.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-asort.o: $(CCANDIR)/ccan/asort/asort.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
+ccan-autodata.o: $(CCANDIR)/ccan/autodata/autodata.c
+	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-ptr_valid.o: $(CCANDIR)/ccan/ptr_valid/ptr_valid.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-read_write_all.o: $(CCANDIR)/ccan/read_write_all/read_write_all.c
@@ -981,8 +677,6 @@ ccan-cdump.o: $(CCANDIR)/ccan/cdump/cdump.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-strmap.o: $(CCANDIR)/ccan/strmap/strmap.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
-ccan-strset.o: $(CCANDIR)/ccan/strset/strset.c
-	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-crypto-siphash24.o: $(CCANDIR)/ccan/crypto/siphash24/siphash24.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-htable.o: $(CCANDIR)/ccan/htable/htable.c
@@ -1027,9 +721,3 @@ ccan-json_escape.o: $(CCANDIR)/ccan/json_escape/json_escape.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 ccan-json_out.o: $(CCANDIR)/ccan/json_out/json_out.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
-ccan-closefrom.o: $(CCANDIR)/ccan/closefrom/closefrom.c
-	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
-ccan-rune-rune.o: $(CCANDIR)/ccan/rune/rune.c
-	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
-ccan-rune-coding.o: $(CCANDIR)/ccan/rune/coding.c
-	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)

README.md

@@ -1,6 +1,6 @@
-# Core Lightning (CLN): A specification compliant Lightning Network implementation in C
+# c-lightning: A specification compliant Lightning Network implementation in C
 
-Core Lightning (previously c-lightning) is a lightweight, highly customizable and [standard compliant][std] implementation of the Lightning Network protocol.
+c-lightning is a lightweight, highly customizable and [standard compliant][std] implementation of the Lightning Network protocol.
 
 * [Getting Started](#getting-started)
     * [Installation](#installation)
@@ -15,52 +15,53 @@ Core Lightning (previously c-lightning) is a lightweight, highly customizable an
     * [Pruning](#pruning)
     * [HD wallet encryption](#hd-wallet-encryption)
 	* [Developers](#developers)
-* [Documentation](https://docs.corelightning.org/docs)
 
 ## Project Status
 
-[![Continuous Integration][actions-badge]][actions]
-[![Pull Requests Welcome][prs-badge]][prs]
-[![Documentation Status][docs-badge]][docs]
-[![BoL2][bol2-badge]][bol2]
-[![Telegram][telegram-badge]][telegram]
-[![Discord][discord-badge]][discord]
-[![Irc][IRC-badge]][IRC]
+[![Build Status][travis-ci]][travis-ci-link]
+[![Pull Requests Welcome][prs]][prs-link]
+[![Irc][IRC]][IRC-link]
+[![Documentation Status](https://readthedocs.org/projects/lightning/badge/?version=docs)][docs]
 
 This implementation has been in production use on the Bitcoin mainnet since early 2018, with the launch of the [Blockstream Store][blockstream-store-blog].
-We recommend getting started by experimenting on `testnet` (`testnet4` or `regtest`), but the implementation is considered stable and can be safely used on mainnet.
-
-## Reach Out to Us
+We recommend getting started by experimenting on `testnet` (or `regtest`), but the implementation is considered stable and can be safely used on mainnet.
 
 Any help testing the implementation, reporting bugs, or helping with outstanding issues is very welcome.
-Don't hesitate to reach out to us on [Build-on-L2][bol2], or on the implementation-specific [mailing list][ml1], or on [CLN Discord][discord], or on [CLN Telegram][telegram], or on IRC at [dev][irc1]/[gen][irc2] channel.
+Don't hesitate to reach out to us on IRC at [#lightning-dev @ freenode.net][irc1], [#c-lightning @ freenode.net][irc2], or on the implementation-specific mailing list [c-lightning@lists.ozlabs.org][ml1], or on the Lightning Network-wide mailing list [lightning-dev@lists.linuxfoundation.org][ml2].
 
 ## Getting Started
 
-Core Lightning only works on Linux and macOS, and requires a locally (or remotely) running `bitcoind` (version 25.0 or above) that is fully caught up with the network you're running on, and relays transactions (ie with `blocksonly=0`).
+c-lightning only works on Linux and Mac OS, and requires a locally (or remotely) running `bitcoind` (version 0.16 or above) that is fully caught up with the network you're running on, and relays transactions (ie with `blocksonly=0`).
 Pruning (`prune=n` option in `bitcoin.conf`) is partially supported, see [here](#pruning) for more details.
 
 ### Installation
 
-There are 3 supported installation options:
+There are 4 supported installation options:
 
- - Installation of a pre-compiled binary from the [release page][releases] on GitHub.
- - Using one of the [provided docker images][dockerhub] on the Docker Hub.
- - Compiling the source code yourself as described in the [installation documentation](doc/getting-started/getting-started/installation.md).
+ - Installation from the [Ubuntu PPA][ppa]
+ - Installation of a pre-compiled binary from the [release page][releases] on Github
+ - Using one of the [provided docker images][dockerhub] on the Docker Hub
+ - Compiling the source code yourself as described in the [installation documentation](doc/INSTALL.md).
+
+For the impatient here's the gist of it for Ubuntu:
+
+```bash
+sudo apt-get install -y software-properties-common
+sudo add-apt-repository -u ppa:bitcoin/bitcoin
+sudo add-apt-repository -u ppa:lightningnetwork/ppa
+sudo apt-get install bitcoind lightningd
+```
 
 ### Starting `lightningd`
 
-#### Regtest (local, fast-start) Option
 If you want to experiment with `lightningd`, there's a script to set
 up a `bitcoind` regtest test network of two local lightning nodes,
-which provides a convenient `start_ln` helper. See the notes at the top
-of the `startup_regtest.sh` file for details on how to use it.
+which provides a convenient `start_ln` helper:
 
 ```bash
 . contrib/startup_regtest.sh
 ```
 
-#### Mainnet Option
 To test with real bitcoin,  you will need to have a local `bitcoind` node running:
 
 ```bash
@@ -78,11 +79,11 @@ You can start `lightningd` with the following command:
 lightningd --network=bitcoin --log-level=debug
 ```
 
-This creates a `.lightning/` subdirectory in your home directory: see `man -l doc/lightningd.8` (or https://docs.corelightning.org/docs) for more runtime options.
+This creates a `.lightning/` subdirectory in your home directory: see `man -l doc/lightningd.8` (or https://lightning.readthedocs.io/) for more runtime options.
 
 ### Using The JSON-RPC Interface
 
-Core Lightning exposes a [JSON-RPC 2.0][jsonrpcspec] interface over a Unix Domain socket; the `lightning-cli` tool can be used to access it, or there is a [python client library](contrib/pyln-client).
+c-lightning exposes a [JSON-RPC 2.0][jsonrpcspec] interface over a Unix Domain socket; the `lightning-cli` tool can be used to access it, or there is a [python client library](contrib/pylightning).
 
 You can use `lightning-cli help` to print a table of RPC methods; `lightning-cli help <command>`
 will offer specific information on that command.
@@ -103,13 +104,18 @@ Once you've started for the first time, there's a script called
 `contrib/bootstrap-node.sh` which will connect you to other nodes on
 the lightning network.
 
-There are also numerous plugins available for Core Lightning which add
-capabilities: in particular there's a collection at: https://github.com/lightningd/plugins
+There are also numerous plugins available for c-lightning which add
+capabilities: in particular there's a collection at:
+
+	https://github.com/lightningd/plugins
+
+Including [helpme][helpme-github] which guides you through setting up
+your first channels and customizing your node.
 
 For a less reckless experience, you can encrypt the HD wallet seed:
  see [HD wallet encryption](#hd-wallet-encryption).
 
-You can also chat to other users at Discord [core-lightning][discord];
+You can also chat to other users at [#c-lightning @ freenode.net][irc2];
 we are always happy to help you get started!
 
 
@@ -125,11 +131,11 @@ lightning-cli newaddr
 
 `lightningd` will register the funds once the transaction is confirmed.
 
-Alternatively you can generate a taproot address should your source of funds support it:
+You may need to generate a p2sh-segwit address if the faucet does not support bech32:
 
 ```bash
-# Return a taproot address
-lightning-cli newaddr p2tr
+# Return a p2sh-segwit address
+lightning-cli newaddr p2sh-segwit
 ```
 
 Confirm `lightningd` got funds by:
@@ -148,7 +154,8 @@ lightning-cli connect <node_id> <ip> [<port>]
 lightning-cli fundchannel <node_id> <amount_in_satoshis>
 ```
 
-This opens a connection and, on top of that connection, then opens a channel.
+This opens a connection and, on top of that connection, then opens
+a channel.
 The funding transaction needs 3 confirmation in order for the channel to be usable, and 6 to be announced for others to use.
 You can check the status of the channel using `lightning-cli listpeers`, which after 3 confirmations (1 on testnet) should say that `state` is `CHANNELD_NORMAL`; after 6 confirmations you can use `lightning-cli listchannels` to verify that the `public` field is now `true`.
 
@@ -165,7 +172,7 @@ lightning-cli invoice <amount> <label> <description>
 
 This returns some internal details, and a standard invoice string called `bolt11` (named after the [BOLT #11 lightning spec][BOLT11]).
 
-[BOLT11]: https://github.com/lightning/bolts/blob/master/11-payment-encoding.md
+[BOLT11]: https://github.com/lightningnetwork/lightning-rfc/blob/master/11-payment-encoding.md
 
 The sender can feed this `bolt11` string to the `decodepay` command to see what it is, and pay it simply using the `pay` command:
 
@@ -184,49 +191,43 @@ Command line options will always override the values in the configuration file.
 To use a configuration file, create a file named `config` within your top-level lightning directory or network subdirectory
 (eg. `~/.lightning/config` or `~/.lightning/bitcoin/config`).  See `man -l doc/lightningd-config.5`.
 
-A sample configuration file is available at `contrib/config-example`.
-
 ## Further information
 
 ### Pruning
 
-Core Lightning requires JSON-RPC access to a fully synchronized `bitcoind` in order to synchronize with the Bitcoin network.
+c-lightning requires JSON-RPC access to a fully synchronized `bitcoind` in order to synchronize with the Bitcoin network.
 Access to ZeroMQ is not required and `bitcoind` does not need to be run with `txindex` like other implementations.
 The lightning daemon will poll `bitcoind` for new blocks that it hasn't processed yet, thus synchronizing itself with `bitcoind`.
-If `bitcoind` prunes a block that Core Lightning has not processed yet, e.g., Core Lightning was not running for a prolonged period, then `bitcoind` will not be able to serve the missing blocks, hence Core Lightning will not be able to synchronize anymore and will be stuck.
-In order to avoid this situation you should be monitoring the gap between Core Lightning's blockheight using `lightning-cli getinfo` and `bitcoind`'s blockheight using `bitcoin-cli getblockchaininfo`.
+If `bitcoind` prunes a block that c-lightning has not processed yet, e.g., c-lightning was not running for a prolonged period, then `bitcoind` will not be able to serve the missing blocks, hence c-lightning will not be able to synchronize anymore and will be stuck.
+In order to avoid this situation you should be monitoring the gap between c-lightning's blockheight using `lightning-cli getinfo` and `bitcoind`'s blockheight using `bitcoin-cli getblockchaininfo`.
 If the two blockheights drift apart it might be necessary to intervene.
 
 ### HD wallet encryption
 
 You can encrypt the `hsm_secret` content (which is used to derive the HD wallet's master key) by passing the `--encrypted-hsm` startup argument, or by using the `hsmtool` (which you can find in the `tool/` directory at the root of this repo) with the `encrypt` method. You can unencrypt an encrypted `hsm_secret` using the `hsmtool` with the `decrypt` method.
 
-If you encrypt your `hsm_secret`, you will have to pass the `--encrypted-hsm` startup option to `lightningd`. Once your `hsm_secret` is encrypted, you __will not__ be able to access your funds without your password, so please beware with your password management. Also, beware of not feeling too safe with an encrypted `hsm_secret`: unlike for `bitcoind` where the wallet encryption can restrict the usage of some RPC command, `lightningd` always needs to access keys from the wallet which is thus __not locked__ (yet), even with an encrypted BIP32 master seed.
+If you encrypt your `hsm_secret`, you will have to pass the `--encrypted-hsm` startup option to `lightningd`. Once your `hsm_secret` is encrypted, you __will not__ be able to access your funds without your password, so please beware with your password management. Also beware of not feeling too safe with an encrypted `hsm_secret`: unlike for `bitcoind` where the wallet encryption can restrict the usage of some RPC command, `lightningd` always need to access keys from the wallet which is thus __not locked__ (yet), even with an encrypted BIP32 master seed.
 
 ### Developers
 
-Developers wishing to contribute should start with the developer guide [here](doc/contribute-to-core-lightning/coding-style-guidelines.md).
+Developers wishing to contribute should start with the developer guide [here](doc/HACKING.md).
+You should also configure with `--enable-developer` to get additional checks and options.
 
 [blockstream-store-blog]: https://blockstream.com/2018/01/16/en-lightning-charge/
-[std]: https://github.com/lightning/bolts
-[prs-badge]: https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat
-[prs]: http://makeapullrequest.com
-[bol2-badge]: https://badgen.net/badge/BoL2/chat/blue
-[bol2]: https://community.corelightning.org
+[std]: https://github.com/lightningnetwork/lightning-rfc
+[travis-ci]: https://travis-ci.org/ElementsProject/lightning.svg?branch=master
+[travis-ci-link]: https://travis-ci.org/ElementsProject/lightning
+[prs]: https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat
+[prs-link]: http://makeapullrequest.com
+[IRC]: https://img.shields.io/badge/chat-on%20freenode-brightgreen.svg
+[IRC-link]: https://webchat.freenode.net/?channels=c-lightning
+[irc1]: http://webchat.freenode.net/?channels=%23lightning-dev
+[irc2]: http://webchat.freenode.net/?channels=%23c-lightning
 [ml1]: https://lists.ozlabs.org/listinfo/c-lightning
-[discord-badge]: https://badgen.net/badge/Discord/chat/blue
-[discord]: https://discord.gg/mE9s4rc5un
-[telegram-badge]: https://badgen.net/badge/Telegram/chat/blue
-[telegram]: https://t.me/lightningd
-[IRC-badge]: https://img.shields.io/badge/IRC-chat-blue.svg
-[IRC]: https://web.libera.chat/#c-lightning
-[irc1]: https://web.libera.chat/#lightning-dev
-[irc2]: https://web.libera.chat/#c-lightning
-[docs-badge]: https://readthedocs.org/projects/lightning/badge/?version=docs
-[docs]: https://docs.corelightning.org/docs
+[ml2]: https://lists.linuxfoundation.org/mailman/listinfo/lightning-dev
+[docs]: https://lightning.readthedocs.org
+[ppa]: https://launchpad.net/~lightningnetwork/+archive/ubuntu/ppa
 [releases]: https://github.com/ElementsProject/lightning/releases
 [dockerhub]: https://hub.docker.com/r/elementsproject/lightningd/
 [jsonrpcspec]: https://www.jsonrpc.org/specification
 [helpme-github]: https://github.com/lightningd/plugins/tree/master/helpme
-[actions-badge]: https://github.com/ElementsProject/lightning/workflows/Continuous%20Integration/badge.svg
-[actions]: https://github.com/ElementsProject/lightning/actions

SECURITY.md

@@ -1,35 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-We have a 3 month release cycle, and the last two versions are supported.
-
-## Reporting a Vulnerability
-
-To report security vulnerabilities, please send an email to one of the following addresses:
-- `rusty@rustcorp.com.au`
-- `security@blockstream.com`
-
-Note: These email addresses are exclusively for vulnerability reporting.
-
-For all other inquiries/communication, please refer to the [Reach Out to Us](https://github.com/ElementsProject/lightning?tab=readme-ov-file#reach-out-to-us) section in our README.
-
-## Signatures For Releases
-
-The following keys may be used to communicate sensitive information to
-developers, and to validate signatures on releases:
-
-| Name | Email | Fingerprint |
-|------|-------|-------------|
-| Blockstream Security Reporting | `security@blockstream.com` | 1176 542D A98E 71E1 3372  2EF7 4AC8 CC88 6844 A2D6 |
-| Rusty Russell | `rusty@rustcorp.com.au` | 15EE 8D6C AB0E 7F0C F999  BFCB D920 0E6C D1AD B8F1 |
-| Christian Decker | `decker@blockstream.com` | B731 AAC5 21B0 1385 9313  F674 A26D 6D9F E088 ED58 |
-| Lisa Neigut | `niftynei@gmail.com` | 30DE 693A E0DE 9E37 B3E7  EB6B BFF0 F678 10C1 EED1 |
-| Alex Myers | `alex@endothermic.dev` | 0437 4E42 789B BBA9 462E  4767 F3BF 63F2 7474 36AB |
-| Peter Neuroth | `pet.v.ne@gmail.com` | 653B 19F3 3DF7 EFF3 E9D1  C94C C3F2 1EE3 87FF 4CD2 |
-| Shahana Farooqui | `sfarooqui@blockstream.com` | FE13 58EB 7793 51DB 24E5  555A A327 573C 9758 9BF5 |
-| Blockstream CLN Release | `cln@blockstream.com` | 616C 52F9 9D06 12B2 A151  B107 4129 A994 AA7E 9852 |
-
-You can import a key by running the following command with that individual’s fingerprint: 
-`gpg --keyserver hkps://keys.openpgp.org --recv-keys "<fingerprint>"`. 
-Ensure that you put quotes around fingerprints containing spaces.

bitcoin/Makefile

@@ -38,18 +38,20 @@ BITCOIN_HEADERS := bitcoin/address.h		\
 	bitcoin/tx_parts.h			\
 	bitcoin/varint.h
 
+check-source: $(BITCOIN_SRC:%=check-src-include-order/%)		\
+	$(BITCOIN_HEADERS:%=check-hdr-include-order/%)
+
 # Bitcoin objects depends on bitcoin/ external/ and ccan
 $(BITCOIN_OBJS): $(CCAN_HEADERS) $(BITCOIN_HEADERS) $(EXTERNAL_HEADERS)
 
-ALL_C_HEADERS += $(BITCOIN_HEADERS)
-ALL_C_SOURCES += $(BITCOIN_SRC)
+check-source-bolt: $(BITCOIN_SRC:%=bolt-check/%) $(BITCOIN_HEADERS:%=bolt-check/%)
 
 check-makefile: check-bitcoin-makefile
 
 check-bitcoin-makefile:
 	@if [ "`echo bitcoin/*.h`" != "$(BITCOIN_HEADERS)" ]; then echo BITCOIN_HEADERS incorrect; exit 1; fi
 
-check-whitespace: check-whitespace/bitcoin/Makefile
+check-whitespace: $(BITCOIN_SRC:%=check-whitespace/%) $(BITCOIN_HEADERS:%=check-whitespace/%) check-whitespace/bitcoin/Makefile
 
 clean: bitcoin-clean
 

bitcoin/address.h

@@ -2,6 +2,7 @@
 #define LIGHTNING_BITCOIN_ADDRESS_H
 #include "config.h"
 #include <ccan/crypto/ripemd160/ripemd160.h>
+#include <ccan/short_types/short_types.h>
 
 /* An address is the RIPEMD160 of the SHA of the public key. */
 struct bitcoin_address {

bitcoin/base58.c

@@ -3,13 +3,16 @@
 // Copyright (c) 2009-2012 The Bitcoin Developers
 // Distributed under the MIT/X11 software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#include "config.h"
-#include <bitcoin/address.h>
-#include <bitcoin/base58.h>
-#include <bitcoin/privkey.h>
-#include <bitcoin/pubkey.h>
-#include <bitcoin/shadouble.h>
+#include "address.h"
+#include "base58.h"
+#include "privkey.h"
+#include "pubkey.h"
+#include "shadouble.h"
+#include <assert.h>
+#include <ccan/build_assert/build_assert.h>
+#include <ccan/tal/str/str.h>
 #include <common/utils.h>
+#include <string.h>
 #include <wally_core.h>
 
 static char *to_base58(const tal_t *ctx, u8 version,
@@ -21,14 +24,13 @@ static char *to_base58(const tal_t *ctx, u8 version,
 	buf[0] = version;
 	memcpy(buf + 1, rmd, sizeof(*rmd));
 
-	tal_wally_start();
-	if (wally_base58_from_bytes((const unsigned char *) buf,
-				    total_length, BASE58_FLAG_CHECKSUM, &out)
-	    != WALLY_OK)
-		out = NULL;
-	tal_wally_end_onto(ctx, out, char);
-
-	return out;
+	if (wally_base58_from_bytes((const unsigned char *) buf, total_length, BASE58_FLAG_CHECKSUM, &out) != WALLY_OK) {
+		return NULL;
+	}else{
+		char *res = tal_strdup(ctx, out);
+		wally_free_string(out);
+		return res;
+	}
 }
 
 char *bitcoin_to_base58(const tal_t *ctx, const struct chainparams *chainparams,
@@ -47,14 +49,16 @@ static bool from_base58(u8 *version,
 			struct ripemd160 *rmd,
 			const char *base58, size_t base58_len)
 {
-	/* Initialize to avoid memcheck complaining if decoding a short value */
-	u8 buf[1 + sizeof(*rmd) + 4] = { 0 };
-	const size_t buflen = sizeof(buf);
-	const uint32_t flags = BASE58_FLAG_CHECKSUM;
+	u8 buf[1 + sizeof(*rmd) + 4];
+	/* Avoid memcheck complaining if decoding resulted in a short value */
+	size_t buflen = sizeof(buf);
+	memset(buf, 0, buflen);
+	char *terminated_base58 = tal_dup_arr(NULL, char, base58, base58_len, 1);
+	terminated_base58[base58_len] = '\0';
 
 	size_t written = 0;
-	int r = wally_base58_n_to_bytes(base58, base58_len, flags,
-					buf, buflen, &written);
+	int r = wally_base58_to_bytes(terminated_base58, BASE58_FLAG_CHECKSUM, buf, buflen, &written);
+	tal_free(terminated_base58);
 	if (r != WALLY_OK || written > buflen) {
 		return false;
 	}
@@ -63,8 +67,63 @@ static bool from_base58(u8 *version,
 	return true;
 }
 
+bool bitcoin_from_base58(u8 *version, struct bitcoin_address *addr,
+			 const char *base58, size_t len)
+{
+	return from_base58(version, &addr->addr, base58, len);
+}
+
+
+bool p2sh_from_base58(u8 *version, struct ripemd160 *p2sh, const char *base58,
+		      size_t len)
+{
+
+	return from_base58(version, p2sh, base58, len);
+}
+
 bool ripemd160_from_base58(u8 *version, struct ripemd160 *rmd,
 			   const char *base58, size_t base58_len)
 {
 	return from_base58(version, rmd, base58, base58_len);
 }
+
+bool key_from_base58(const char *base58, size_t base58_len,
+		     bool *test_net, struct privkey *priv, struct pubkey *key)
+{
+	// 1 byte version, 32 byte private key, 1 byte compressed, 4 byte checksum
+	u8 keybuf[1 + 32 + 1 + 4];
+	char *terminated_base58 = tal_dup_arr(NULL, char, base58, base58_len, 1);
+	terminated_base58[base58_len] = '\0';
+	size_t keybuflen = sizeof(keybuf);
+
+
+	size_t written = 0;
+	int r = wally_base58_to_bytes(terminated_base58, BASE58_FLAG_CHECKSUM, keybuf, keybuflen, &written);
+	wally_bzero(terminated_base58, base58_len + 1);
+	tal_free(terminated_base58);
+	if (r != WALLY_OK || written > keybuflen)
+		return false;
+
+	/* Byte after key should be 1 to represent a compressed key. */
+	if (keybuf[1 + 32] != 1)
+		return false;
+
+	if (keybuf[0] == 128)
+		*test_net = false;
+	else if (keybuf[0] == 239)
+		*test_net = true;
+	else
+		return false;
+
+	/* Copy out secret. */
+	memcpy(priv->secret.data, keybuf + 1, sizeof(priv->secret.data));
+
+	if (!secp256k1_ec_seckey_verify(secp256k1_ctx, priv->secret.data))
+		return false;
+
+	/* Get public key, too. */
+	if (!pubkey_from_privkey(priv, key))
+		return false;
+
+	return true;
+}

bitcoin/base58.h

@@ -4,6 +4,10 @@
 
 #include <bitcoin/chainparams.h>
 #include <ccan/crypto/ripemd160/ripemd160.h>
+#include <ccan/short_types/short_types.h>
+#include <ccan/tal/tal.h>
+#include <stdbool.h>
+#include <stdlib.h>
 
 struct pubkey;
 struct privkey;
@@ -12,10 +16,17 @@ struct bitcoin_address;
 /* Bitcoin address encoded in base58, with version and checksum */
 char *bitcoin_to_base58(const tal_t *ctx, const struct chainparams *chainparams,
 			const struct bitcoin_address *addr);
+bool bitcoin_from_base58(u8 *version, struct bitcoin_address *addr,
+			 const char *base58, size_t len);
 
 /* P2SH address encoded as base58, with version and checksum */
 char *p2sh_to_base58(const tal_t *ctx, const struct chainparams *chainparams,
 		     const struct ripemd160 *p2sh);
+bool p2sh_from_base58(u8 *version, struct ripemd160 *p2sh, const char *base58,
+		      size_t len);
+
+bool key_from_base58(const char *base58, size_t base58_len,
+		     bool *test_net, struct privkey *priv, struct pubkey *key);
 
 /* Decode a p2pkh or p2sh into the ripemd160 hash */
 bool ripemd160_from_base58(u8 *version, struct ripemd160 *rmd,

bitcoin/block.c

@@ -1,11 +1,11 @@
-#include "config.h"
 #include <assert.h>
 #include <bitcoin/block.h>
 #include <bitcoin/chainparams.h>
 #include <bitcoin/tx.h>
 #include <ccan/mem/mem.h>
 #include <ccan/str/hex/hex.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
+#include <wire/wire.h>
 
 /* Sets *cursor to NULL and returns NULL when a pull fails. */
 static const u8 *pull(const u8 **cursor, size_t *max, void *copy, size_t n)
@@ -208,11 +208,9 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams,
 
 	num = pull_varint(&p, &len);
 	b->tx = tal_arr(b, struct bitcoin_tx *, num);
-	b->txids = tal_arr(b, struct bitcoin_txid, num);
 	for (i = 0; i < num; i++) {
-		b->tx[i] = pull_bitcoin_tx_only(b->tx, &p, &len);
+		b->tx[i] = pull_bitcoin_tx(b->tx, &p, &len);
 		b->tx[i]->chainparams = chainparams;
-		bitcoin_txid(b->tx[i], &b->txids[i]);
 	}
 
 	/* We should end up not overrunning, nor have extra */
@@ -229,22 +227,34 @@ void bitcoin_block_blkid(const struct bitcoin_block *b,
 	*out = b->hdr.hash;
 }
 
-static bool bitcoin_blkid_to_hex(const struct bitcoin_blkid *blockid,
-				 char *hexstr, size_t hexstr_len)
+/* We do the same hex-reversing crud as txids. */
+bool bitcoin_blkid_from_hex(const char *hexstr, size_t hexstr_len,
+			    struct bitcoin_blkid *blockid)
+{
+	struct bitcoin_txid fake_txid;
+	if (!bitcoin_txid_from_hex(hexstr, hexstr_len, &fake_txid))
+		return false;
+	blockid->shad = fake_txid.shad;
+	return true;
+}
+
+bool bitcoin_blkid_to_hex(const struct bitcoin_blkid *blockid,
+			  char *hexstr, size_t hexstr_len)
 {
 	struct bitcoin_txid fake_txid;
 	fake_txid.shad = blockid->shad;
 	return bitcoin_txid_to_hex(&fake_txid, hexstr, hexstr_len);
 }
 
-char *fmt_bitcoin_blkid(const tal_t *ctx,
-			const struct bitcoin_blkid *blkid)
+static char *fmt_bitcoin_blkid(const tal_t *ctx,
+			       const struct bitcoin_blkid *blkid)
 {
 	char *hexstr = tal_arr(ctx, char, hex_str_size(sizeof(*blkid)));
 
 	bitcoin_blkid_to_hex(blkid, hexstr, hex_str_size(sizeof(*blkid)));
 	return hexstr;
 }
+REGISTER_TYPE_TO_STRING(bitcoin_blkid, fmt_bitcoin_blkid);
 
 void fromwire_bitcoin_blkid(const u8 **cursor, size_t *max,
 			    struct bitcoin_blkid *blkid)

bitcoin/block.h

@@ -3,8 +3,10 @@
 #include "config.h"
 #include "bitcoin/shadouble.h"
 #include <ccan/endian/endian.h>
+#include <ccan/short_types/short_types.h>
 #include <ccan/structeq/structeq.h>
 #include <ccan/tal/tal.h>
+#include <stdbool.h>
 
 struct chainparams;
 
@@ -34,7 +36,6 @@ struct bitcoin_block {
 	struct bitcoin_block_hdr hdr;
 	/* tal_count shows now many */
 	struct bitcoin_tx **tx;
-	struct bitcoin_txid *txids;
 };
 
 struct bitcoin_block *
@@ -45,6 +46,14 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams,
 void bitcoin_block_blkid(const struct bitcoin_block *block,
 			 struct bitcoin_blkid *out);
 
+/* Parse hex string to get blockid (reversed, a-la bitcoind). */
+bool bitcoin_blkid_from_hex(const char *hexstr, size_t hexstr_len,
+			    struct bitcoin_blkid *blockid);
+
+/* Get hex string of blockid (reversed, a-la bitcoind). */
+bool bitcoin_blkid_to_hex(const struct bitcoin_blkid *blockid,
+			  char *hexstr, size_t hexstr_len);
+
 /* Marshalling/unmarshaling over the wire */
 void towire_bitcoin_blkid(u8 **pptr, const struct bitcoin_blkid *blkid);
 void fromwire_bitcoin_blkid(const u8 **cursor, size_t *max,
@@ -53,7 +62,4 @@ void fromwire_chainparams(const u8 **cursor, size_t *max,
 			  const struct chainparams **chainparams);
 void towire_chainparams(u8 **cursor, const struct chainparams *chainparams);
 
-char *fmt_bitcoin_blkid(const tal_t *ctx,
-			const struct bitcoin_blkid *blkid);
-
 #endif /* LIGHTNING_BITCOIN_BLOCK_H */

bitcoin/chainparams.c

@@ -1,9 +1,8 @@
-#include "config.h"
-#include <assert.h>
-#include <bitcoin/chainparams.h>
+#include "chainparams.h"
 #include <ccan/array_size/array_size.h>
-#include <ccan/tal/str/str.h>
+#include <ccan/str/str.h>
 #include <common/utils.h>
+#include <string.h>
 
 /* Version codes for BIP32 extended keys in libwally-core.
  * Stolen from wally_bip32.h in libwally-core*/
@@ -28,8 +27,7 @@ static u8 liquid_regtest_fee_asset[] = {
 
 const struct chainparams networks[] = {
     {.network_name = "bitcoin",
-     .onchain_hrp = "bc",
-     .lightning_hrp = "bc",
+     .bip173_name = "bc",
      .bip70_name = "main",
      .genesis_blockhash = {{{.u.u8 = {0x6f, 0xe2, 0x8c, 0x0a, 0xb6, 0xf1, 0xb3,
 				      0x72, 0xc1, 0xa6, 0xa2, 0x46, 0xae, 0x63,
@@ -37,7 +35,6 @@ const struct chainparams networks[] = {
 				      0x5a, 0x08, 0x9c, 0x68, 0xd6, 0x19, 0x00,
 				      0x00, 0x00, 0x00, 0x00}}}},
      .rpc_port = 8332,
-     .ln_port = 9735,
      .cli = "bitcoin-cli",
      .cli_args = NULL,
      .cli_min_supported_version = 150000,
@@ -50,7 +47,6 @@ const struct chainparams networks[] = {
       */
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      /* "Lightning Charge Powers Developers & Blockstream Store" */
      .when_lightning_became_cool = 504500,
      .p2pkh_version = 0,
@@ -61,23 +57,20 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_MAIN_PRIVATE},
      .is_elements = false},
     {.network_name = "regtest",
-     .onchain_hrp = "bcrt",
-     .lightning_hrp = "bcrt",
+     .bip173_name = "bcrt",
      .bip70_name = "regtest",
      .genesis_blockhash = {{{.u.u8 = {0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b,
 				      0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb,
 				      0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e,
 				      0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c,
 				      0xf1, 0x88, 0x91, 0x0f}}}},
-     .rpc_port = 18443,
-     .ln_port = 19846,
+     .rpc_port = 18332,
      .cli = "bitcoin-cli",
      .cli_args = "-regtest",
      .cli_min_supported_version = 150000,
      .dust_limit = { 546 },
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 111,
      .p2sh_version = 196,
@@ -87,34 +80,25 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
      .is_elements = false},
     {.network_name = "signet",
-     .onchain_hrp = "tb",
-     .lightning_hrp = "tbs",
+     .bip173_name = "sb",
      .bip70_name = "signet",
-     // 00000008819873e925422c1ff0f99f7cc9bbb232af63a077a480a3633bee1ef6
-     .genesis_blockhash = {{{.u.u8 = {0xf6, 0x1e, 0xee, 0x3b, 0x63, 0xa3, 0x80,
-				      0xa4, 0x77, 0xa0, 0x63, 0xaf, 0x32, 0xb2,
-				      0xbb, 0xc9, 0x7c, 0x9f, 0xf9, 0xf0, 0x1f,
-				      0x2c, 0x42, 0x25, 0xe9, 0x73, 0x98, 0x81,
-				      0x08, 0x00, 0x00, 0x00}}}},
+     .genesis_blockhash = {{{.u.u8 = {0xce, 0xbd, 0x8f, 0x6e, 0x69, 0x77, 0x30, 0xb7, 0x4c, 0x70, 0x9c, 0xdd, 0x1e, 0x6a, 0xba, 0xaf, 0x2a, 0xfc, 0x98, 0xbf, 0x4c, 0xff, 0xb2, 0x39, 0xf3, 0xdb, 0x44, 0x27, 0x64, 0x29, 0x00, 0x00}}}},
      .rpc_port = 38332,
-     .ln_port = 39735,
      .cli = "bitcoin-cli",
      .cli_args = "-signet",
      .cli_min_supported_version = 150000,
      .dust_limit = { 546 },
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
-     .p2pkh_version = 111,
-     .p2sh_version = 196,
+     .p2pkh_version = 125,
+     .p2sh_version = 87,
      .testnet = true,
      .bip32_key_version = {.bip32_pubkey_version = BIP32_VER_SIGT_PUBLIC, .bip32_privkey_version = BIP32_VER_SIGT_PRIVATE},
      .is_elements = false,
     },
     {.network_name = "testnet",
-     .onchain_hrp = "tb",
-     .lightning_hrp = "tb",
+     .bip173_name = "tb",
      .bip70_name = "test",
      .genesis_blockhash = {{{.u.u8 = {0x43, 0x49, 0x7f, 0xd7, 0xf8, 0x26, 0x95,
 				      0x71, 0x08, 0xf4, 0xa3, 0x0f, 0xd9, 0xce,
@@ -122,40 +106,12 @@ const struct chainparams networks[] = {
 				      0xe9, 0x0e, 0xad, 0x01, 0xea, 0x33, 0x09,
 				      0x00, 0x00, 0x00, 0x00}}}},
      .rpc_port = 18332,
-     .ln_port = 19735,
      .cli = "bitcoin-cli",
      .cli_args = "-testnet",
      .cli_min_supported_version = 150000,
      .dust_limit = { 546 },
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
-     .p2pkh_version = 111,
-     .p2sh_version = 196,
-     .testnet = true,
-     .fee_asset_tag = NULL,
-     .bip32_key_version = {.bip32_pubkey_version = BIP32_VER_TEST_PUBLIC,
-			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
-     .is_elements = false},
-    {.network_name = "testnet4",
-     .onchain_hrp = "tb",
-     .lightning_hrp = "tb",
-     .bip70_name = "testnet4",
-     // 00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043
-     .genesis_blockhash = {{{.u.u8 = {0x43, 0xf0, 0x8b, 0xda, 0xb0, 0x50, 0xe3,
-				      0x5b, 0x56, 0x7c, 0x86, 0x4b, 0x91, 0xf4,
-				      0x7f, 0x50, 0xae, 0x72, 0x5a, 0xe2, 0xde,
-				      0x53, 0xbc, 0xfb, 0xba, 0xf2, 0x84, 0xda,
-				      0x00, 0x00, 0x00, 0x00}}}},
-     .rpc_port = 48332,
-     .ln_port = 49735,
-     .cli = "bitcoin-cli",
-     .cli_args = "-testnet4",
-     .cli_min_supported_version = 150000,
-     .dust_limit = { 546 },
-     .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
-     .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .p2pkh_version = 111,
      .p2sh_version = 196,
      .testnet = true,
@@ -164,8 +120,7 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
      .is_elements = false},
     {.network_name = "litecoin",
-     .onchain_hrp = "ltc",
-     .lightning_hrp = "ltc",
+     .bip173_name = "ltc",
      .bip70_name = "main",
      .genesis_blockhash = {{{.u.u8 = {0xe2, 0xbf, 0x04, 0x7e, 0x7e, 0x5a, 0x19,
 				      0x1a, 0xa4, 0xef, 0x34, 0xd3, 0x14, 0x97,
@@ -173,14 +128,12 @@ const struct chainparams networks[] = {
 				      0x1e, 0xda, 0xba, 0x59, 0x40, 0xfd, 0x1f,
 				      0xe3, 0x65, 0xa7, 0x12}}}},
      .rpc_port = 9332,
-     .ln_port = 9735,
      .cli = "litecoin-cli",
      .cli_args = NULL,
      .cli_min_supported_version = 150000,
      .dust_limit = { 100000 },
      .max_funding = AMOUNT_SAT_INIT(60 * ((1 << 24) - 1)),
      .max_payment = AMOUNT_MSAT_INIT(60 * 0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1320000,
      .p2pkh_version = 48,
      .p2sh_version = 50,
@@ -190,8 +143,7 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_MAIN_PRIVATE},
      .is_elements = false},
     {.network_name = "litecoin-testnet",
-     .onchain_hrp = "tltc",
-     .lightning_hrp = "tltc",
+     .bip173_name = "tltc",
      .bip70_name = "test",
      .genesis_blockhash = {{{.u.u8 = {0xa0, 0x29, 0x3e, 0x4e, 0xeb, 0x3d, 0xa6,
 				      0xe6, 0xf5, 0x6f, 0x81, 0xed, 0x59, 0x5f,
@@ -199,14 +151,12 @@ const struct chainparams networks[] = {
 				      0x13, 0xee, 0xfd, 0xd9, 0x51, 0x28, 0x4b,
 				      0x5a, 0x62, 0x66, 0x49}}}},
      .rpc_port = 19332,
-     .ln_port = 9735,
      .cli = "litecoin-cli",
      .cli_args = "-testnet",
      .cli_min_supported_version = 150000,
      .dust_limit = { 100000 },
      .max_funding = AMOUNT_SAT_INIT(60 * ((1 << 24) - 1)),
      .max_payment = AMOUNT_MSAT_INIT(60 * 0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 111,
      .p2sh_version = 58,
@@ -216,8 +166,7 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
      .is_elements = false},
     {.network_name = "liquid-regtest",
-     .onchain_hrp = "ert",
-     .lightning_hrp = "ert",
+     .bip173_name = "ert",
      .bip70_name = "liquid-regtest",
      .genesis_blockhash = {{{.u.u8 = {0x9f, 0x87, 0xeb, 0x58, 0x0b, 0x9e, 0x5f,
 				      0x11, 0xdc, 0x21, 0x1e, 0x9f, 0xb6, 0x6a,
@@ -225,13 +174,11 @@ const struct chainparams networks[] = {
 				      0xfe, 0x14, 0x68, 0x01, 0x16, 0x23, 0x93,
 				      0x36, 0x42, 0x86, 0xc6}}}},
      .rpc_port = 19332,
-     .ln_port = 20735,
      .cli = "elements-cli",
      .cli_args = "-chain=liquid-regtest",
      .dust_limit = {546},
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 91,
      .p2sh_version = 75,
@@ -241,8 +188,7 @@ const struct chainparams networks[] = {
 			   .bip32_privkey_version = BIP32_VER_TEST_PRIVATE},
      .is_elements = true},
     {.network_name = "liquid",
-     .onchain_hrp = "ex",
-     .lightning_hrp = "ex",
+     .bip173_name = "ex",
      .bip70_name = "liquidv1",
      .genesis_blockhash = {{{.u.u8 = {0x14, 0x66, 0x27, 0x58, 0x36, 0x22, 0x0d,
 				      0xb2, 0x94, 0x4c, 0xa0, 0x59, 0xa3, 0xa1,
@@ -250,13 +196,11 @@ const struct chainparams networks[] = {
 				      0x68, 0x8d, 0x2c, 0x37, 0x92, 0x96, 0x88,
 				      0x8a, 0x20, 0x60, 0x03}}}},
      .rpc_port = 7041,
-     .ln_port = 9735,
      .cli = "elements-cli",
      .cli_args = "-chain=liquidv1",
      .dust_limit = {546},
      .max_funding = AMOUNT_SAT_INIT((1 << 24) - 1),
      .max_payment = AMOUNT_MSAT_INIT(0xFFFFFFFFULL),
-     .max_supply = AMOUNT_SAT_INIT(2100000000000000),
      .when_lightning_became_cool = 1,
      .p2pkh_version = 57,
      .p2sh_version = 39,
@@ -277,6 +221,14 @@ const struct chainparams *chainparams_for_network(const char *network_name)
 	return NULL;
 }
 
+const struct chainparams **chainparams_for_networks(const tal_t *ctx)
+{
+	const struct chainparams **params = tal_arr(ctx, const struct chainparams*, 0);
+	for (size_t i = 0; i < ARRAY_SIZE(networks); i++)
+		tal_arr_expand(&params, &networks[i]);
+	return params;
+}
+
 const struct chainparams *chainparams_by_chainhash(const struct bitcoin_blkid *chain_hash)
 {
 	for (size_t i = 0; i < ARRAY_SIZE(networks); i++) {
@@ -287,10 +239,10 @@ const struct chainparams *chainparams_by_chainhash(const struct bitcoin_blkid *c
 	return NULL;
 }
 
-const struct chainparams *chainparams_by_lightning_hrp(const char *lightning_hrp)
+const struct chainparams *chainparams_by_bip173(const char *bip173_name)
 {
 	for (size_t i = 0; i < ARRAY_SIZE(networks); i++) {
-		if (streq(lightning_hrp, networks[i].lightning_hrp)) {
+		if (streq(bip173_name, networks[i].bip173_name)) {
 			return &networks[i];
 		}
 	}
@@ -305,8 +257,3 @@ const char *chainparams_get_network_names(const tal_t *ctx)
     return networks_string;
 }
 
-int chainparams_get_ln_port(const struct chainparams *params)
-{
-	assert(params);
-	return params->ln_port;
-}

bitcoin/chainparams.h

@@ -3,35 +3,22 @@
 
 #include "config.h"
 #include <bitcoin/block.h>
+#include <ccan/short_types/short_types.h>
+#include <ccan/tal/str/str.h>
 #include <common/amount.h>
 #include <common/bip32.h>
+#include <stdbool.h>
 
 #define ELEMENTS_ASSET_LEN 33
 
 struct chainparams {
 	const char *network_name;
-	/* Unfortunately starting with signet, we now have diverging
-	 * conventions for the "BIP173" Human Readable Part (HRP).
-	 * On onchain signet, the HRP is `tb` , but on Lightning
-	 * signet the HRP is `tbs`.
-	 */
-	const char *onchain_hrp;
-	const char *lightning_hrp;
+	const char *bip173_name;
 	/*'bip70_name' is corresponding to the 'chain' field of
 	 * the API 'getblockchaininfo' */
 	const char *bip70_name;
 	const struct bitcoin_blkid genesis_blockhash;
 	const int rpc_port;
-	/**
-	 * BOLT 1:
-	 *
-	 * The default TCP port depends on the network used. The most common networks are:
-	 *
-	 * - Bitcoin mainet with port number 9735 or the corresponding hexadecimal `0x2607`;
-	 * - Bitcoin testnet with port number 19735 (`0x4D17`);
-	 * - Bitcoin signet with port number 39735 (`0x9B37`).
-	 */
-	const int ln_port;
 	const char *cli;
 	const char *cli_args;
 	/* The min numeric version of cli supported */
@@ -39,8 +26,6 @@ struct chainparams {
 	const struct amount_sat dust_limit;
 	const struct amount_sat max_funding;
 	const struct amount_msat max_payment;
-	/* Total coins in network */
-	const struct amount_sat max_supply;
 	const u32 when_lightning_became_cool;
 	const u8 p2pkh_version;
 	const u8 p2sh_version;
@@ -59,12 +44,18 @@ struct chainparams {
  */
 const struct chainparams *chainparams_for_network(const char *network_name);
 
+/**
+ * chainparams_for_networks - Get blockchain parameters for all known networks,
+ *                            as a tal array.
+ */
+const struct chainparams **chainparams_for_networks(const tal_t *ctx);
+
 /**
  * chainparams_by_bip173 - Helper to get a network by its bip173 name
  *
  * This lets us decode BOLT11 addresses.
  */
-const struct chainparams *chainparams_by_lightning_hrp(const char *lightning_hrp);
+const struct chainparams *chainparams_by_bip173(const char *bip173_name);
 
 /**
  * chainparams_by_chainhash - Helper to get a network by its genesis blockhash
@@ -76,9 +67,4 @@ const struct chainparams *chainparams_by_chainhash(const struct bitcoin_blkid *c
  */
 const char *chainparams_get_network_names(const tal_t *ctx);
 
-/**
- * chainparams_get_ln_port - Return the lightning network default port by
- * network if the chainparams is initialized, otherwise 9735 as mock port
- */
-int chainparams_get_ln_port(const struct chainparams *params);
 #endif /* LIGHTNING_BITCOIN_CHAINPARAMS_H */

bitcoin/feerate.c

@@ -1,12 +1,9 @@
-#include "config.h"
-#include <assert.h>
 #include <bitcoin/feerate.h>
+#include <limits.h>
+#include <stdlib.h>
 
 u32 feerate_from_style(u32 feerate, enum feerate_style style)
 {
-	/* Make sure it's called somewhere! */
-	assert(feerate_floor_check() == FEERATE_FLOOR);
-
 	switch (style) {
 	case FEERATE_PER_KSIPA:
 		return feerate;

bitcoin/feerate.h

@@ -39,7 +39,7 @@ enum feerate_style {
 	FEERATE_PER_KBYTE
 };
 
-static inline u32 feerate_floor_check(void)
+static inline u32 feerate_floor(void)
 {
 	/* Assert that bitcoind will see this as above minRelayTxFee */
 	BUILD_ASSERT(FEERATE_BITCOIND_SEES(FEERATE_FLOOR, MINIMUM_TX_WEIGHT)

bitcoin/locktime.c

@@ -1,7 +1,7 @@
-#include "config.h"
 #include <assert.h>
 #include <bitcoin/locktime.h>
 #include <ccan/tal/str/str.h>
+#include <common/type_to_string.h>
 
 #define SECONDS_POINT 500000000
 
@@ -22,13 +22,60 @@ static bool abs_is_seconds(u32 locktime)
 	return locktime >= SECONDS_POINT;
 }
 
+bool rel_locktime_is_seconds(const struct rel_locktime *rel)
+{
+	return rel->locktime & BIP68_SECONDS_FLAG;
+}
+
+u32 rel_locktime_to_seconds(const struct rel_locktime *rel)
+{
+	assert(rel_locktime_is_seconds(rel));
+	return (rel->locktime & BIP68_LOCKTIME_MASK) << BIP68_SECONDS_SHIFT;
+}
+
+u32 rel_locktime_to_blocks(const struct rel_locktime *rel)
+{
+	assert(!rel_locktime_is_seconds(rel));
+	return rel->locktime & BIP68_LOCKTIME_MASK;
+}
+
 bool blocks_to_abs_locktime(u32 blocks, struct abs_locktime *abs)
 {
 	return abs_blocks_to_locktime(blocks, &abs->locktime);
 }
 
-u32 abs_locktime_to_blocks(const struct abs_locktime *abs)
+bool abs_locktime_is_seconds(const struct abs_locktime *abs)
 {
-	assert(!abs_is_seconds(abs->locktime));
+	return abs_is_seconds(abs->locktime);
+}
+
+u32 abs_locktime_to_seconds(const struct abs_locktime *abs)
+{
+	assert(abs_locktime_is_seconds(abs));
 	return abs->locktime;
 }
+
+u32 abs_locktime_to_blocks(const struct abs_locktime *abs)
+{
+	assert(!abs_locktime_is_seconds(abs));
+	return abs->locktime;
+}
+
+static char *fmt_rel_locktime(const tal_t *ctx, const struct rel_locktime *rl)
+{
+	if (rel_locktime_is_seconds(rl))
+		return tal_fmt(ctx, "+%usec", rel_locktime_to_seconds(rl));
+	else
+		return tal_fmt(ctx, "+%ublocks", rel_locktime_to_blocks(rl));
+}
+
+static char *fmt_abs_locktime(const tal_t *ctx, const struct abs_locktime *al)
+{
+	if (abs_locktime_is_seconds(al))
+		return tal_fmt(ctx, "%usec", abs_locktime_to_seconds(al));
+	else
+		return tal_fmt(ctx, "%ublocks", abs_locktime_to_blocks(al));
+}
+
+REGISTER_TYPE_TO_STRING(rel_locktime, fmt_rel_locktime);
+REGISTER_TYPE_TO_STRING(abs_locktime, fmt_abs_locktime);

bitcoin/locktime.h

@@ -4,12 +4,23 @@
 #include <ccan/short_types/short_types.h>
 #include <stdbool.h>
 
+/* As used by nSequence and OP_CHECKSEQUENCEVERIFY (BIP68) */
+struct rel_locktime {
+	u32 locktime;
+};
+
+bool rel_locktime_is_seconds(const struct rel_locktime *rel);
+u32 rel_locktime_to_seconds(const struct rel_locktime *rel);
+u32 rel_locktime_to_blocks(const struct rel_locktime *rel);
+
 /* As used by nLocktime and OP_CHECKLOCKTIMEVERIFY (BIP65) */
 struct abs_locktime {
 	u32 locktime;
 };
 
 bool blocks_to_abs_locktime(u32 blocks, struct abs_locktime *abs);
+bool abs_locktime_is_seconds(const struct abs_locktime *abs);
+u32 abs_locktime_to_seconds(const struct abs_locktime *abs);
 u32 abs_locktime_to_blocks(const struct abs_locktime *abs);
 
 #endif /* LIGHTNING_BITCOIN_LOCKTIME_H */

bitcoin/preimage.c

@@ -1,6 +1,4 @@
-#include "config.h"
 #include <bitcoin/preimage.h>
-#include <common/utils.h>
 #include <wire/wire.h>
 
 void fromwire_preimage(const u8 **cursor, size_t *max, struct preimage *preimage)
@@ -13,7 +11,4 @@ void towire_preimage(u8 **pptr, const struct preimage *preimage)
 	towire(pptr, preimage, sizeof(*preimage));
 }
 
-char *fmt_preimage(const tal_t *ctx, const struct preimage *preimage)
-{
-	return tal_hexstr(ctx, preimage, sizeof(*preimage));
-}
+

bitcoin/preimage.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/short_types/short_types.h>
 #include <ccan/structeq/structeq.h>
-#include <ccan/tal/tal.h>
 
 struct preimage {
 	u8 r[32];
@@ -14,5 +13,4 @@ STRUCTEQ_DEF(preimage, 0, r);
 void fromwire_preimage(const u8 **cursor, size_t *max, struct preimage *preimage);
 void towire_preimage(u8 **pptr, const struct preimage *preimage);
 
-char *fmt_preimage(const tal_t *ctx, const struct preimage *preimage);
 #endif /* LIGHTNING_BITCOIN_PREIMAGE_H */

bitcoin/privkey.c

@@ -1,10 +1,11 @@
-#include "config.h"
-#include <bitcoin/privkey.h>
+#include "privkey.h"
+#include <assert.h>
+#include <ccan/mem/mem.h>
 #include <ccan/str/hex/hex.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
-char *fmt_privkey(const tal_t *ctx, const struct privkey *secret)
+static char *privkey_to_hexstr(const tal_t *ctx, const struct privkey *secret)
 {
 	/* Bitcoin appends "01" to indicate the pubkey is compressed. */
 	char *str = tal_arr(ctx, char, hex_str_size(sizeof(*secret) + 1));
@@ -12,11 +13,8 @@ char *fmt_privkey(const tal_t *ctx, const struct privkey *secret)
 	strcat(str, "01");
 	return str;
 }
-
-char *fmt_secret(const tal_t *ctx, const struct secret *secret)
-{
-	return tal_hexstr(ctx, secret, sizeof(*secret));
-}
+REGISTER_TYPE_TO_STRING(privkey, privkey_to_hexstr);
+REGISTER_TYPE_TO_HEXSTR(secret);
 
 bool secret_eq_consttime(const struct secret *a, const struct secret *b)
 {

bitcoin/privkey.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/short_types/short_types.h>
 #include <ccan/structeq/structeq.h>
-#include <ccan/tal/tal.h>
 
 #define PRIVKEY_LEN 32
 
@@ -26,7 +25,4 @@ void fromwire_privkey(const u8 **cursor, size_t *max, struct privkey *privkey);
 void towire_privkey(u8 **pptr, const struct privkey *privkey);
 void towire_secret(u8 **pptr, const struct secret *secret);
 
-char *fmt_privkey(const tal_t *ctx, const struct privkey *privkey);
-char *fmt_secret(const tal_t *ctx, const struct secret *secret);
-
 #endif /* LIGHTNING_BITCOIN_PRIVKEY_H */

bitcoin/psbt.h

@@ -3,75 +3,24 @@
 #include "config.h"
 #include <ccan/short_types/short_types.h>
 #include <ccan/tal/tal.h>
+#include <stddef.h>
 
+struct wally_tx_input;
+struct wally_tx_output;
 struct wally_psbt;
 struct wally_psbt_input;
 struct wally_tx;
-struct wally_tx_input;
-struct wally_tx_output;
-struct wally_map;
 struct amount_asset;
 struct amount_sat;
-struct bitcoin_outpoint;
 struct bitcoin_signature;
 struct bitcoin_txid;
 struct pubkey;
 
+void psbt_destroy(struct wally_psbt *psbt);
 
-/* Utility we need for psbt stuffs;
- * add the varint onto the given array */
-void add_varint(u8 **arr, size_t val);
-
-/**
- * create_psbt - Create a new psbt object
- *
- * @ctx - allocation context
- * @num_inputs - number of inputs to allocate
- * @num_outputs - number of outputs to allocate
- * @locktime - locktime for the transaction
- */
-struct wally_psbt *create_psbt(const tal_t *ctx, size_t num_inputs, size_t num_outputs, u32 locktime);
-
-/*
- * new_psbt - Create a PSBT, using the passed in tx
- * 	      as the locktime/inputs/output psbt fields
- *
- * @ctx - allocation context
- * @wtx - global_tx starter kit
- */
 struct wally_psbt *new_psbt(const tal_t *ctx,
 			    const struct wally_tx *wtx);
 
-/**
- * clone_psbt - Clone a PSBT onto passed in context
- *
- * @ctx - allocation context
- * @psbt - psbt to be cloned
- */
-struct wally_psbt *clone_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
-
-/**
- * combine_psbt - Combine two PSBT into a cloned copy
- *
- * @ctx - allocation context
- * @psbt0 - one psbt
- * @psbt1 - other psbt
- */
-struct wally_psbt *combine_psbt(const tal_t *ctx,
-				const struct wally_psbt *psbt0,
-				const struct wally_psbt *psbt1);
-
-/**
- * audit_psbt - Audit the memory structure of the PSBT.
- *
- * This checks all known memory allocations in the PSBT and asserts that they
- * are all allocated with 'ctx' being it's parent.
- *
- * ctx - the ctx all memory *should* be attached to
- * psbt - the PSBT to audit.
- * */
-void audit_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
-
 /**
  * psbt_is_finalized - Check if tx is ready to be extracted
  *
@@ -85,78 +34,19 @@ bool psbt_is_finalized(const struct wally_psbt *psbt);
 
 /**
  * psbt_txid - get the txid of the psbt (what it would be after finalization)
- * @ctx: the context to allocate wtx off, if *@wtx isn't NULL.
  * @psbt: the psbt.
  * @txid: the transaction id (output)
  * @wtx: if non-NULL, returns a copy of the transaction (caller must wally_tx_free).
  */
-void psbt_txid(const tal_t *ctx,
-	       const struct wally_psbt *psbt, struct bitcoin_txid *txid,
+void psbt_txid(const struct wally_psbt *psbt, struct bitcoin_txid *txid,
 	       struct wally_tx **wtx);
 
-/* psbt_elements_normalize_fees - Figure out the fee output for a PSET
- *
- * Adds a fee output if not present, or updates it to include the diff
- * between inputs - outputs. Unlike bitcoin, elements requires every
- * satoshi to be accounted for in an output.
- */
-void psbt_elements_normalize_fees(struct wally_psbt *psbt);
-
-/**
- * psbt_finalize - finalize this psbt.
- *
- * Returns false if we can't, otherwise returns true and psbt_is_finalized()
- * is true.
- */
-bool psbt_finalize(struct wally_psbt *psbt);
-
-/**
- * psbt_final_tx - extract transaction from finalized psbt.
- * @ctx: context to tallocate return
- * @psbt: psbt to extract.
- *
- * If @psbt isn't final, or we can't extract tx, returns NULL.
- */
-struct wally_tx *psbt_final_tx(const tal_t *ctx, const struct wally_psbt *psbt);
-
-/* psbt_make_key - Create a new, proprietary Core Lightning key
- *
- * @ctx - allocation context
- * @key_subtype - type for this key
- * @key_data - any extra data to append to the key
- *
- * Returns a proprietary-prefixed key.
- */
-u8 *psbt_make_key(const tal_t *ctx, u8 key_subtype, const u8 *key_data);
+struct wally_tx *psbt_finalize(struct wally_psbt *psbt, bool finalize_in_place);
 
 struct wally_psbt_input *psbt_add_input(struct wally_psbt *psbt,
-					const struct wally_tx_input *input,
+					struct wally_tx_input *input,
 					size_t insert_at);
 
-/* One stop shop for adding an input + metadata to a PSBT */
-struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt,
-					   const struct bitcoin_outpoint *outpoint,
-					   u32 sequence,
-					   const u8 *scriptSig,
-					   const u8 *input_wscript,
-					   const u8 *redeemscript);
-
-/* psbt_input_set_wit_utxo - Set the witness_utxo field for this PSBT */
-void psbt_input_set_wit_utxo(struct wally_psbt *psbt, size_t in,
-			     const u8 *scriptPubkey, struct amount_sat amt);
-
-/* psbt_input_set_utxo - Set the non-witness utxo field for this PSBT input */
-void psbt_input_set_utxo(struct wally_psbt *psbt, size_t in,
-			 const struct wally_tx *prev_tx);
-
-void psbt_input_set_outpoint(struct wally_psbt *psbt, size_t in,
-			     struct bitcoin_outpoint outpoint);
-
-/* psbt_elements_input_set_asset - Set the asset/value fields for an
- * 				   Elements PSBT (PSET, technically */
-void psbt_elements_input_set_asset(struct wally_psbt *psbt, size_t in,
-				   struct amount_asset *asset);
-
 void psbt_rm_input(struct wally_psbt *psbt,
 		   size_t remove_at);
 
@@ -164,171 +54,42 @@ struct wally_psbt_output *psbt_add_output(struct wally_psbt *psbt,
 					  struct wally_tx_output *output,
 					  size_t insert_at);
 
-/**
- * wally_psbt_output - Append a new output to the PSBT
- *
- * @psbt - PSBT to append output to
- * @script - scriptPubKey of the output
- * @amount - value of the output
- */
-struct wally_psbt_output *psbt_append_output(struct wally_psbt *psbt,
-					     const u8 *script,
-					     struct amount_sat amount);
-struct wally_psbt_output *psbt_insert_output(struct wally_psbt *psbt,
-					     const u8 *script,
-					     struct amount_sat amount,
-					     size_t insert_at);
-
 void psbt_rm_output(struct wally_psbt *psbt,
 		    size_t remove_at);
 
 void psbt_input_add_pubkey(struct wally_psbt *psbt, size_t in,
-			   const struct pubkey *pubkey, bool is_taproot);
+			   const struct pubkey *pubkey);
 
 WARN_UNUSED_RESULT bool psbt_input_set_signature(struct wally_psbt *psbt, size_t in,
 						 const struct pubkey *pubkey,
 						 const struct bitcoin_signature *sig);
 
-/* Returns false on error. On success, *signature_found is set to true if the
- * input has a signature present for `pubkey` and false if if one was not found.
- * Only signature presence is checked, it is not validated. */
-WARN_UNUSED_RESULT bool psbt_input_have_signature(const struct wally_psbt *psbt,
-						  size_t in,
-						  const struct pubkey *pubkey,
-						  bool *signature_found);
-
-/* Returns false on error. On success *sig is set to the signature otherwise
- * *sig is set to NULL. */
-WARN_UNUSED_RESULT bool psbt_input_get_ecdsa_sig(const tal_t *ctx,
-						 const struct wally_psbt *psbt,
-						 size_t in,
-						 const struct pubkey *pubkey,
-						 struct bitcoin_signature **sig);
-
-void psbt_input_set_witscript(struct wally_psbt *psbt, size_t in, const u8 *wscript);
-
-/* psbt_input_set_unknown - Set the given Key-Value in the psbt's input keymap
- * @ctx - tal context for allocations
- * @in - psbt input to set key-value on
- * @key - key for key-value pair
- * @value - value to set
- * @value_len - length of {@value}
- */
-void psbt_input_set_unknown(const tal_t *ctx,
-			    struct wally_psbt_input *in,
-			    const u8 *key,
-			    const void *value,
-			    size_t value_len);
-
-/* psbt_get_lightning - Fetch a proprietary lightning value from the given map
- *
- * @map - map of unknowns to search for key
- * @proprietary_type - type no. to look for
- * @val_len - (out) length of value (if found)
- *
- * Returns: value of type {proprietary_type}, or NULL if not found */
-void *psbt_get_lightning(const struct wally_map *map,
-			 const u8 proprietary_type,
-			 size_t *val_len);
-
-/* psbt_set_lightning - Set a propreitary lightning value on the given map
- *
- * @map - map of unknowns to set the value
- * @proprietary_type - type no. to set
- * @value - the value to be set
- * @val_len - length of value
- */
-void psbt_set_lightning(const tal_t *ctx,
-			struct wally_map *map,
-			const u8 proprietary_type,
-			const void *value,
-			size_t val_len);
-
-/* psbt_output_set_unknown - Set the given Key-Value in the psbt's output keymap
- *
- * @ctx - tal context for allocations
- * @out - psbt output to set key-value on
- * @key - key for key-value pair
- * @value - value to set
- * @value_len - length of {@value}
- */
-void psbt_output_set_unknown(const tal_t *ctx,
-			     struct wally_psbt_output *out,
-			     const u8 *key, const void *value,
-			     size_t value_len);
-
-/* psbt_input_get_amount - Returns the value of this input
- *
- * @psbt - psbt
- * @in - index of input whose value you're returning
- * */
-struct amount_sat psbt_input_get_amount(const struct wally_psbt *psbt,
+void psbt_input_set_prev_utxo(struct wally_psbt *psbt,
+			      size_t in,
+			      const u8 *wscript,
+			      struct amount_sat amt);
+void psbt_input_set_prev_utxo_wscript(struct wally_psbt *psbt, size_t in,
+			              const u8 *wscript,
+				      struct amount_sat amt);
+void psbt_elements_input_init(struct wally_psbt *psbt, size_t in,
+			      const u8 *scriptPubkey,
+			      struct amount_asset *asset,
+			      const u8 *nonce);
+void psbt_elements_input_init_witness(struct wally_psbt *psbt, size_t in,
+				      const u8 *witscript,
+				      struct amount_asset *asset,
+				      const u8 *nonce);
+bool psbt_input_set_redeemscript(struct wally_psbt *psbt, size_t in,
+				 const u8 *redeemscript);
+struct amount_sat psbt_input_get_amount(struct wally_psbt *psbt,
 					size_t in);
 
-/* psbt_input_get_weight - Calculate the tx weight for input index `in` */
-size_t psbt_input_get_weight(const struct wally_psbt *psbt,
-			     size_t in);
-
-/* psbt_output_get_amount - Returns the value of this output
- *
- * @psbt - psbt
- * @out -index of output whose value you're returning
- */
-struct amount_sat psbt_output_get_amount(const struct wally_psbt *psbt,
-					 size_t out);
-
-/* psbt_output_get_weight - Calculate the tx weight for output index `outnum` */
-size_t psbt_output_get_weight(const struct wally_psbt *psbt,
-			      size_t outnum);
-
-/* psbt_compute_fee - Returns value of fee for PSBT
- *
- * @psbt -psbt
- */
-struct amount_sat psbt_compute_fee(const struct wally_psbt *psbt);
-
-/* psbt_has_input - Is this input present on this psbt
- *
- * @psbt - psbt
- * @outpoint - txid/index spent by input
- */
-bool psbt_has_input(const struct wally_psbt *psbt,
-		    const struct bitcoin_outpoint *outpoint);
-
-/* wally_psbt_input_spends - Returns true if PSBT input spends given outpoint
- *
- * @input - psbt input
- * @outpoint - outpoint
- */
-bool wally_psbt_input_spends(const struct wally_psbt_input *input,
-               const struct bitcoin_outpoint *outpoint);
-
-void wally_psbt_input_get_outpoint(const struct wally_psbt_input *in,
-                 struct bitcoin_outpoint *outpoint);
-
-const u8 *wally_psbt_output_get_script(const tal_t *ctx,
-                     const struct wally_psbt_output *output);
-
-void wally_psbt_input_get_txid(const struct wally_psbt_input *in,
-                 struct bitcoin_txid *txid);
-
-struct amount_asset
-wally_psbt_output_get_amount(const struct wally_psbt_output *output);
-
-/* psbt_set_version - Returns false if there was any issue with the PSBT.
- * Returns true if it was a well-formed PSET and treats it as a no-op
- */
-bool psbt_set_version(struct wally_psbt *psbt, u32 version);
-
-bool elements_psbt_output_is_fee(const struct wally_psbt *psbt, size_t outnum);
-
 struct wally_psbt *psbt_from_b64(const tal_t *ctx,
 				 const char *b64,
 				 size_t b64len);
-char *fmt_wally_psbt(const tal_t *ctx, const struct wally_psbt *psbt);
+char *psbt_to_b64(const tal_t *ctx, const struct wally_psbt *psbt);
 const u8 *psbt_get_bytes(const tal_t *ctx, const struct wally_psbt *psbt,
 			 size_t *bytes_written);
-bool validate_psbt(const struct wally_psbt *psbt);
 struct wally_psbt *psbt_from_bytes(const tal_t *ctx, const u8 *bytes,
 				   size_t byte_len);
 void towire_wally_psbt(u8 **pptr, const struct wally_psbt *psbt);

bitcoin/pubkey.c

@@ -1,9 +1,9 @@
-#include "config.h"
+#include "privkey.h"
+#include "pubkey.h"
 #include <assert.h>
-#include <bitcoin/privkey.h>
-#include <bitcoin/pubkey.h>
 #include <ccan/mem/mem.h>
 #include <ccan/str/hex/hex.h>
+#include <common/type_to_string.h>
 #include <common/utils.h>
 #include <wire/wire.h>
 
@@ -62,15 +62,16 @@ bool pubkey_from_hexstr(const char *derstr, size_t slen, struct pubkey *key)
 	return pubkey_from_der(der, dlen, key);
 }
 
-char *fmt_pubkey(const tal_t *ctx, const struct pubkey *key)
+char *pubkey_to_hexstr(const tal_t *ctx, const struct pubkey *key)
 {
 	unsigned char der[PUBKEY_CMPR_LEN];
 
 	pubkey_to_der(der, key);
 	return tal_hexstr(ctx, der, sizeof(der));
 }
+REGISTER_TYPE_TO_STRING(pubkey, pubkey_to_hexstr);
 
-char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key)
+char *secp256k1_pubkey_to_hexstr(const tal_t *ctx, const secp256k1_pubkey *key)
 {
 	unsigned char der[PUBKEY_CMPR_LEN];
 	size_t outlen = sizeof(der);
@@ -80,10 +81,11 @@ char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key)
 	assert(outlen == sizeof(der));
 	return tal_hexstr(ctx, der, sizeof(der));
 }
+REGISTER_TYPE_TO_STRING(secp256k1_pubkey, secp256k1_pubkey_to_hexstr);
 
 int pubkey_cmp(const struct pubkey *a, const struct pubkey *b)
 {
-	u8 keya[PUBKEY_CMPR_LEN], keyb[PUBKEY_CMPR_LEN];
+	u8 keya[33], keyb[33];
 	pubkey_to_der(keya, a);
 	pubkey_to_der(keyb, b);
 	return memcmp(keya, keyb, sizeof(keya));

bitcoin/pubkey.h

@@ -2,10 +2,11 @@
 #define LIGHTNING_BITCOIN_PUBKEY_H
 #include "config.h"
 #include <ccan/crypto/ripemd160/ripemd160.h>
+#include <ccan/crypto/sha256/sha256.h>
 #include <ccan/short_types/short_types.h>
 #include <ccan/structeq/structeq.h>
 #include <ccan/tal/tal.h>
-#include <secp256k1_extrakeys.h>
+#include <secp256k1.h>
 
 struct privkey;
 struct secret;
@@ -23,8 +24,10 @@ STRUCTEQ_DEF(pubkey, 0, pubkey.data);
 bool pubkey_from_hexstr(const char *derstr, size_t derlen, struct pubkey *key);
 
 /* Convert from hex string of DER (scriptPubKey from validateaddress) */
-char *fmt_pubkey(const tal_t *ctx, const struct pubkey *key);
-char *fmt_secp256k1_pubkey(const tal_t *ctx, const secp256k1_pubkey *key);
+char *pubkey_to_hexstr(const tal_t *ctx, const struct pubkey *key);
+
+/* Convenience wrapper for a raw secp256k1_pubkey */
+char *secp256k1_pubkey_to_hexstr(const tal_t *ctx, const secp256k1_pubkey *key);
 
 /* Point from secret */
 bool pubkey_from_secret(const struct secret *secret, struct pubkey *key);
@@ -42,7 +45,7 @@ void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN], const struct pubkey *key);
 /* Compare the keys `a` and `b`. Return <0 if `a`<`b`, 0 if equal and >0 otherwise */
 int pubkey_cmp(const struct pubkey *a, const struct pubkey *b);
 
-/* If the two pubkeys[] are id1 and id2, which index would id1 be? */
+/* If the two nodes[] are id1 and id2, which index would id1 be? */
 static inline int pubkey_idx(const struct pubkey *id1, const struct pubkey *id2)
 {
 	return pubkey_cmp(id1, id2) > 0;

bitcoin/script.c

@@ -1,17 +1,44 @@
-#include "config.h"
+#include "address.h"
+#include "locktime.h"
+#include "preimage.h"
+#include "pubkey.h"
+#include "script.h"
 #include <assert.h>
-#include <bitcoin/address.h>
-#include <bitcoin/locktime.h>
-#include <bitcoin/preimage.h>
-#include <bitcoin/pubkey.h>
-#include <bitcoin/script.h>
+#include <ccan/crypto/ripemd160/ripemd160.h>
+#include <ccan/crypto/sha256/sha256.h>
 #include <ccan/endian/endian.h>
 #include <ccan/mem/mem.h>
 #include <common/utils.h>
-#include <sodium/randombytes.h>
 
-/* To push 0-75 bytes onto stack. */
+/* Some standard ops */
+#define OP_0		0x00
 #define OP_PUSHBYTES(val) (val)
+#define OP_PUSHDATA1	0x4C
+#define OP_PUSHDATA2	0x4D
+#define OP_PUSHDATA4	0x4E
+#define OP_NOP		0x61
+#define OP_IF		0x63
+#define OP_NOTIF	0x64
+#define OP_ELSE		0x67
+#define OP_ENDIF	0x68
+#define OP_RETURN	0x6a
+#define OP_2DROP	0x6d
+#define OP_IFDUP	0x73
+#define OP_DEPTH	0x74
+#define OP_DROP		0x75
+#define OP_DUP		0x76
+#define OP_SWAP		0x7c
+#define OP_EQUAL	0x87
+#define OP_EQUALVERIFY	0x88
+#define OP_SIZE		0x82
+#define OP_1SUB		0x8C
+#define OP_ADD		0x93
+#define OP_CHECKSIG	0xAC
+#define OP_CHECKSIGVERIFY	0xAD
+#define OP_CHECKMULTISIG	0xAE
+#define OP_HASH160	0xA9
+#define OP_CHECKSEQUENCEVERIFY	0xB2
+#define OP_CHECKLOCKTIMEVERIFY	0xB1
 
 /* Bitcoin's OP_HASH160 is RIPEMD(SHA256()) */
 static void hash160(struct ripemd160 *redeemhash, const void *mem, size_t len)
@@ -187,14 +214,11 @@ u8 *scriptpubkey_p2pkh(const tal_t *ctx, const struct bitcoin_address *addr)
 	return script;
 }
 
-u8 *scriptpubkey_opreturn_padded(const tal_t *ctx)
+u8 *scriptpubkey_opreturn(const tal_t *ctx)
 {
 	u8 *script = tal_arr(ctx, u8, 0);
-	u8 random[20];
-	randombytes_buf(random, sizeof(random));
 
 	add_op(&script, OP_RETURN);
-	script_push_bytes(&script, random, sizeof(random));
 	return script;
 }
 
@@ -226,30 +250,18 @@ u8 *bitcoin_redeem_p2sh_p2wpkh(const tal_t *ctx, const struct pubkey *key)
 	return script;
 }
 
-u8 *bitcoin_scriptsig_redeem(const tal_t *ctx,
-			     const u8 *redeemscript TAKES)
+u8 *bitcoin_scriptsig_p2sh_p2wpkh(const tal_t *ctx, const struct pubkey *key)
 {
-	u8 *script;
+	u8 *redeemscript = bitcoin_redeem_p2sh_p2wpkh(ctx, key), *script;
 
 	/* BIP141: The scriptSig must be exactly a push of the BIP16
 	 * redeemScript or validation fails. */
 	script = tal_arr(ctx, u8, 0);
-	script_push_bytes(&script, redeemscript,
-			  tal_count(redeemscript));
-
-	if (taken(redeemscript))
-		tal_free(redeemscript);
-
+	script_push_bytes(&script, redeemscript, tal_count(redeemscript));
+	tal_free(redeemscript);
 	return script;
 }
 
-u8 *bitcoin_scriptsig_p2sh_p2wpkh(const tal_t *ctx, const struct pubkey *key)
-{
-	u8 *redeemscript =
-		bitcoin_redeem_p2sh_p2wpkh(NULL, key);
-	return bitcoin_scriptsig_redeem(ctx, take(redeemscript));
-}
-
 u8 **bitcoin_witness_p2wpkh(const tal_t *ctx,
 			    const struct bitcoin_signature *sig,
 			    const struct pubkey *key)
@@ -310,119 +322,39 @@ u8 *scriptpubkey_witness_raw(const tal_t *ctx, u8 version,
 	return script;
 }
 
-u8 *scriptpubkey_raw_p2tr(const tal_t *ctx, const struct pubkey *output_pubkey)
-{
-	int ok;
-	secp256k1_xonly_pubkey x_key;
-	unsigned char x_key_bytes[32];
-	u8 *script = tal_arr(ctx, u8, 0);
-
-	add_op(&script, OP_1);
-
-	ok = secp256k1_xonly_pubkey_from_pubkey(secp256k1_ctx,
-		&x_key,
-		/* pk_parity */ NULL,
-		&(output_pubkey->pubkey));
-	assert(ok);
-
-	ok = secp256k1_xonly_pubkey_serialize(secp256k1_ctx,
-		x_key_bytes,
-		&x_key);
-	assert(ok);
-
-	script_push_bytes(&script, x_key_bytes, sizeof(x_key_bytes));
-	assert(tal_count(script) == BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
-	return script;
-}
-
-u8 *scriptpubkey_raw_p2tr_derkey(const tal_t *ctx, const u8 output_der[33])
-{
-	struct pubkey tr_key;
-	if (!pubkey_from_der(output_der, 33, &tr_key)) {
-		abort();
-	}
-	return scriptpubkey_raw_p2tr(ctx, &tr_key);
-}
-
-u8 *scriptpubkey_p2tr(const tal_t *ctx, const struct pubkey *inner_pubkey)
-{
-	unsigned char key_bytes[33];
-	unsigned char tweaked_key_bytes[33];
-	size_t out_len = sizeof(key_bytes);
-	u8 *script = tal_arr(ctx, u8, 0);
-
-	add_op(&script, OP_1);
-
-	secp256k1_ec_pubkey_serialize(secp256k1_ctx, key_bytes, &out_len, &inner_pubkey->pubkey, SECP256K1_EC_COMPRESSED);
-	/* Only commit to inner pubkey in tweak */
-	if (wally_ec_public_key_bip341_tweak(key_bytes, 33, /* merkle_root*/ NULL, 0, 0 /* flags */, tweaked_key_bytes, sizeof(tweaked_key_bytes)) != WALLY_OK)
-		abort();
-
-	/* Cut off the first byte from the serialized compressed key */
-	script_push_bytes(&script, tweaked_key_bytes + 1, sizeof(tweaked_key_bytes) - 1);
-	assert(tal_count(script) == BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
-	return script;
-}
-
-u8 *scriptpubkey_p2tr_derkey(const tal_t *ctx, const u8 inner_der[33])
-{
-	struct pubkey tr_key;
-	if (!pubkey_from_der(inner_der, 33, &tr_key)) {
-		abort();
-	}
-	return scriptpubkey_p2tr(ctx, &tr_key);
-}
-
-/* BOLT #3:
+/* BOLT-a12da24dd0102c170365124782b46d9710950ac1 #3:
  *
  * #### `to_remote` Output
  *
- * If `option_anchors` applies to the commitment
+ * If `option_anchor_outputs` applies to the commitment
  * transaction, the `to_remote` output is encumbered by a one
  * block csv lock.
- *    <remotepubkey> OP_CHECKSIGVERIFY 1 OP_CHECKSEQUENCEVERIFY
+ *    <remote_pubkey> OP_CHECKSIGVERIFY 1 OP_CHECKSEQUENCEVERIFY
  */
-/* BOLT- #3
- * ##### Leased channel (`option_will_fund`)
- *
- * If a `lease` applies to the channel, the `to_remote` output
- * of the `initiator` ensures the `leasor` funds are not
- * spendable until the lease expires.
- *
- * <remote_pubkey> OP_CHECKSIGVERIFY MAX(1, lease_end - blockheight) OP_CHECKSEQUENCEVERIFY
- */
-
-u8 *bitcoin_wscript_to_remote_anchored(const tal_t *ctx,
-				       const struct pubkey *remote_key,
-				       u32 csv_lock)
+u8 *anchor_to_remote_redeem(const tal_t *ctx,
+			    const struct pubkey *remote_key)
 {
 	u8 *script = tal_arr(ctx, u8, 0);
 	add_push_key(&script, remote_key);
 	add_op(&script, OP_CHECKSIGVERIFY);
-	add_number(&script, csv_lock);
+	add_number(&script, 1);
 	add_op(&script, OP_CHECKSEQUENCEVERIFY);
 
-	assert(is_to_remote_anchored_witness_script(script, tal_bytelen(script)));
+	assert(is_anchor_witness_script(script, tal_bytelen(script)));
 	return script;
 }
 
-bool is_to_remote_anchored_witness_script(const u8 *script, size_t script_len)
+bool is_anchor_witness_script(const u8 *script, size_t script_len)
 {
-	size_t len = 34 + 1 + 1 + 1;
-	/* With option_will_fund, the pushbytes can be up to 2 bytes more
-	 *
-	 * <remote_pubkey> OP_CHECKSIGVERIFY
-	 * 		MAX(1, lease_end - blockheight)
-	 * 		OP_CHECKSEQUENCEVERIFY
-	 */
-	if (script_len < len || script_len > len + 2)
+	if (script_len != 34 + 1 + 1 + 1)
 		return false;
 	if (script[0] != OP_PUSHBYTES(33))
 		return false;
 	if (script[34] != OP_CHECKSIGVERIFY)
 		return false;
-	/* FIXME: check for push value */
-	if (script[script_len - 1] != OP_CHECKSEQUENCEVERIFY)
+	if (script[35] != 0x51)
+		return false;
+	if (script[36] != OP_CHECKSEQUENCEVERIFY)
 		return false;
 	return true;
 }
@@ -476,8 +408,10 @@ u8 *p2wpkh_scriptcode(const tal_t *ctx, const struct pubkey *key)
 	return script;
 }
 
-bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
+bool is_p2pkh(const u8 *script, struct bitcoin_address *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2PKH_LEN)
 		return false;
 	if (script[0] != OP_DUP)
@@ -495,8 +429,10 @@ bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
 	return true;
 }
 
-bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr)
+bool is_p2sh(const u8 *script, struct ripemd160 *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2SH_LEN)
 		return false;
 	if (script[0] != OP_HASH160)
@@ -510,8 +446,10 @@ bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr)
 	return true;
 }
 
-bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr)
+bool is_p2wsh(const u8 *script, struct sha256 *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2WSH_LEN)
 		return false;
 	if (script[0] != OP_0)
@@ -523,8 +461,10 @@ bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr)
 	return true;
 }
 
-bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr)
+bool is_p2wpkh(const u8 *script, struct bitcoin_address *addr)
 {
+	size_t script_len = tal_count(script);
+
 	if (script_len != BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN)
 		return false;
 	if (script[0] != OP_0)
@@ -536,34 +476,10 @@ bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr
 	return true;
 }
 
-bool is_p2tr(const u8 *script, size_t script_len, u8 xonly_pubkey[32])
+bool is_known_scripttype(const u8 *script)
 {
-	if (script_len != BITCOIN_SCRIPTPUBKEY_P2TR_LEN)
-		return false;
-	if (script[0] != OP_1)
-		return false;
-	/* x-only pubkey */
-	if (script[1] != OP_PUSHBYTES(32))
-		return false;
-	if (xonly_pubkey)
-		memcpy(xonly_pubkey, script+2, 32);
-	return true;
-}
-
-bool is_known_scripttype(const u8 *script, size_t script_len)
-{
-	return is_p2wpkh(script, script_len, NULL)
-		|| is_p2wsh(script, script_len, NULL)
-		|| is_p2sh(script, script_len, NULL)
-		|| is_p2pkh(script, script_len, NULL)
-		|| is_p2tr(script, script_len, NULL);
-}
-
-bool is_known_segwit_scripttype(const u8 *script, size_t script_len)
-{
-	return is_p2wpkh(script, script_len, NULL)
-		|| is_p2wsh(script, script_len, NULL)
-		|| is_p2tr(script, script_len, NULL);
+	return is_p2wpkh(script, NULL) || is_p2wsh(script, NULL)
+		|| is_p2sh(script, NULL) || is_p2pkh(script, NULL);
 }
 
 u8 **bitcoin_witness_sig_and_element(const tal_t *ctx,
@@ -598,28 +514,7 @@ u8 **bitcoin_witness_sig_and_element(const tal_t *ctx,
  *     OP_ENDIF
  *     OP_CHECKSIG
  */
-/* BOLT- #3
- * ##### Leased channel (`option_will_fund`)
- * If a `lease` applies to the channel, the `to_local` output of the `accepter`
- * ensures the `leasor` funds are not spendable until the lease expires.
- *
- * In a leased channel, the `to_local` output that pays the `accepter` node
- * is modified so that its CSV is equal to the greater of the
- * `to_self_delay` or the `lease_end` - `blockheight`.
- *
- *  OP_IF
- *      # Penalty transaction
- *      <revocationpubkey>
- *  OP_ELSE
- *      MAX(`to_self_delay`, `lease_end` - `blockheight`)
- *      OP_CHECKSEQUENCEVERIFY
- *      OP_DROP
- *      <local_delayedpubkey>
- *  OP_ENDIF
- *  OP_CHECKSIG
- */
 u8 *bitcoin_wscript_to_local(const tal_t *ctx, u16 to_self_delay,
-			     u32 lease_remaining,
 			     const struct pubkey *revocation_pubkey,
 			     const struct pubkey *local_delayedkey)
 {
@@ -627,7 +522,7 @@ u8 *bitcoin_wscript_to_local(const tal_t *ctx, u16 to_self_delay,
 	add_op(&script, OP_IF);
 	add_push_key(&script, revocation_pubkey);
 	add_op(&script, OP_ELSE);
-	add_number(&script, max_unsigned(lease_remaining, to_self_delay));
+	add_number(&script, to_self_delay);
 	add_op(&script, OP_CHECKSEQUENCEVERIFY);
 	add_op(&script, OP_DROP);
 	add_push_key(&script, local_delayedkey);
@@ -642,8 +537,7 @@ u8 *bitcoin_wscript_to_local(const tal_t *ctx, u16 to_self_delay,
  *
  * This output sends funds to either an HTLC-timeout transaction after the
  * HTLC-timeout or to the remote node using the payment preimage or the
- * revocation key. The output is a P2WSH, with a witness script (no
- * option_anchors):
+ * revocation key. The output is a P2WSH, with a witness script:
  *
  *     # To remote node with revocation key
  *     OP_DUP OP_HASH160 <RIPEMD160(SHA256(revocationpubkey))> OP_EQUAL
@@ -660,8 +554,9 @@ u8 *bitcoin_wscript_to_local(const tal_t *ctx, u16 to_self_delay,
  *             OP_CHECKSIG
  *         OP_ENDIF
  *     OP_ENDIF
- *
- * Or, with `option_anchors`:
+ */
+/* BOLT-a12da24dd0102c170365124782b46d9710950ac1 #3:
+ * Or, with `option_anchor_outputs`:
  *
  *  # To remote node with revocation key
  *  OP_DUP OP_HASH160 <RIPEMD160(SHA256(revocationpubkey))> OP_EQUAL
@@ -685,8 +580,7 @@ u8 *bitcoin_wscript_htlc_offer_ripemd160(const tal_t *ctx,
 					 const struct pubkey *remotehtlckey,
 					 const struct ripemd160 *payment_ripemd,
 					 const struct pubkey *revocationkey,
-					 bool option_anchor_outputs,
-					 bool option_anchors_zero_fee_htlc_tx)
+					 bool option_anchor_outputs)
 {
 	u8 *script = tal_arr(ctx, u8, 0);
 	struct ripemd160 ripemd;
@@ -718,7 +612,7 @@ u8 *bitcoin_wscript_htlc_offer_ripemd160(const tal_t *ctx,
 	add_op(&script, OP_EQUALVERIFY);
 	add_op(&script, OP_CHECKSIG);
 	add_op(&script, OP_ENDIF);
-	if (option_anchor_outputs || option_anchors_zero_fee_htlc_tx) {
+	if (option_anchor_outputs) {
 		add_number(&script, 1);
 		add_op(&script, OP_CHECKSEQUENCEVERIFY);
 		add_op(&script, OP_DROP);
@@ -733,8 +627,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
 			       const struct pubkey *remotehtlckey,
 			       const struct sha256 *payment_hash,
 			       const struct pubkey *revocationkey,
-			       bool option_anchor_outputs,
-			       bool option_anchors_zero_fee_htlc_tx)
+			       bool option_anchor_outputs)
 {
 	struct ripemd160 ripemd;
 
@@ -742,8 +635,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
 	return bitcoin_wscript_htlc_offer_ripemd160(ctx, localhtlckey,
 						    remotehtlckey,
 						    &ripemd, revocationkey,
-						    option_anchor_outputs,
-						    option_anchors_zero_fee_htlc_tx);
+						    option_anchor_outputs);
 }
 
 /* BOLT #3:
@@ -752,8 +644,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
  *
  * This output sends funds to either the remote node after the HTLC-timeout or
  * using the revocation key, or to an HTLC-success transaction with a
- * successful payment preimage. The output is a P2WSH, with a witness script
- * (no `option_anchors`):
+ * successful payment preimage. The output is a P2WSH, with a witness script:
  *
  *     # To remote node with revocation key
  *     OP_DUP OP_HASH160 <RIPEMD160(SHA256(revocationpubkey))> OP_EQUAL
@@ -772,8 +663,9 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
  *             OP_CHECKSIG
  *         OP_ENDIF
  *     OP_ENDIF
- *
- * Or, with `option_anchors`:
+ */
+/* BOLT-a12da24dd0102c170365124782b46d9710950ac1 #3:
+ * Or, with `option_anchor_outputs`:
  *
  *  # To remote node with revocation key
  *  OP_DUP OP_HASH160 <RIPEMD160(SHA256(revocationpubkey))> OP_EQUAL
@@ -799,8 +691,7 @@ u8 *bitcoin_wscript_htlc_receive_ripemd(const tal_t *ctx,
 					const struct pubkey *remotehtlckey,
 					const struct ripemd160 *payment_ripemd,
 					const struct pubkey *revocationkey,
-					bool option_anchor_outputs,
-					bool option_anchors_zero_fee_htlc_tx)
+					bool option_anchor_outputs)
 {
 	u8 *script = tal_arr(ctx, u8, 0);
 	struct ripemd160 ripemd;
@@ -835,7 +726,7 @@ u8 *bitcoin_wscript_htlc_receive_ripemd(const tal_t *ctx,
 	add_op(&script, OP_DROP);
 	add_op(&script, OP_CHECKSIG);
 	add_op(&script, OP_ENDIF);
-	if (option_anchor_outputs || option_anchors_zero_fee_htlc_tx) {
+	if (option_anchor_outputs) {
 		add_number(&script, 1);
 		add_op(&script, OP_CHECKSEQUENCEVERIFY);
 		add_op(&script, OP_DROP);
@@ -851,8 +742,7 @@ u8 *bitcoin_wscript_htlc_receive(const tal_t *ctx,
 				 const struct pubkey *remotehtlckey,
 				 const struct sha256 *payment_hash,
 				 const struct pubkey *revocationkey,
-				 bool option_anchor_outputs,
-				 bool option_anchors_zero_fee_htlc_tx)
+				 bool option_anchor_outputs)
 {
 	struct ripemd160 ripemd;
 
@@ -860,8 +750,7 @@ u8 *bitcoin_wscript_htlc_receive(const tal_t *ctx,
 	return bitcoin_wscript_htlc_receive_ripemd(ctx, htlc_abstimeout,
 						   localhtlckey, remotehtlckey,
 						   &ripemd, revocationkey,
-						   option_anchor_outputs,
-						   option_anchors_zero_fee_htlc_tx);
+						   option_anchor_outputs);
 }
 
 /* BOLT #3:
@@ -943,8 +832,8 @@ u8 *bitcoin_wscript_anchor(const tal_t *ctx,
 {
 	u8 *script = tal_arr(ctx, u8, 0);
 
-	/* BOLT #3:
-	 * #### `to_local_anchor` and `to_remote_anchor` Output (option_anchors)
+	/* BOLT-a12da24dd0102c170365124782b46d9710950ac1 #3:
+	 * #### `to_local_anchor` and `to_remote_anchor` Output (option_anchor_outputs)
 	 *...
 	 *  <local_funding_pubkey/remote_funding_pubkey> OP_CHECKSIG OP_IFDUP
 	 *  OP_NOTIF
@@ -959,35 +848,15 @@ u8 *bitcoin_wscript_anchor(const tal_t *ctx,
 	add_op(&script, OP_CHECKSEQUENCEVERIFY);
 	add_op(&script, OP_ENDIF);
 
-	assert(is_anchor_witness_script(script, tal_bytelen(script)));
 	return script;
 }
 
-bool is_anchor_witness_script(const u8 *script, size_t script_len)
-{
-	if (script_len != 34 + 1 + 1 + 1 + 1 + 1 + 1)
-		return false;
-	if (script[0] != OP_PUSHBYTES(33))
-		return false;
-	if (script[34] != OP_CHECKSIG)
-		return false;
-	if (script[35] != OP_IFDUP)
-		return false;
-	if (script[36] != OP_NOTIF)
-		return false;
-	if (script[37] != 0x50 + 16)
-		return false;
-	if (script[38] != OP_CHECKSEQUENCEVERIFY)
-		return false;
-	if (script[39] != OP_ENDIF)
-		return false;
-	return true;
-}
-
 bool scripteq(const u8 *s1, const u8 *s2)
 {
-	size_t s1_len = tal_count(s1), s2_len = tal_count(s2);
-	memcheck(s1, s1_len);
-	memcheck(s2, s2_len);
-	return memeq(s1, s1_len, s2, s2_len);
+	memcheck(s1, tal_count(s1));
+	memcheck(s2, tal_count(s2));
+
+	if (tal_count(s1) != tal_count(s2))
+		return false;
+	return memcmp(s1, s2, tal_count(s1)) == 0;
 }

bitcoin/script.h

@@ -3,7 +3,8 @@
 #include "config.h"
 #include "signature.h"
 #include "tx.h"
-#include <wally_script.h>
+#include <ccan/short_types/short_types.h>
+#include <ccan/tal/tal.h>
 
 struct bitcoin_address;
 struct preimage;
@@ -27,11 +28,8 @@ u8 *scriptpubkey_p2sh_hash(const tal_t *ctx, const struct ripemd160 *redeemhash)
 /* Create an output script using p2pkh */
 u8 *scriptpubkey_p2pkh(const tal_t *ctx, const struct bitcoin_address *addr);
 
-/* Create a prunable output script with 20 random bytes.
- * This is needed since a spend from a p2wpkh to an `OP_RETURN` without
- * any other outputs would result in a transaction smaller than the
- * minimum size.  */
-u8 *scriptpubkey_opreturn_padded(const tal_t *ctx);
+/* Create a prunable output script */
+u8 *scriptpubkey_opreturn(const tal_t *ctx);
 
 /* Create an input script which spends p2pkh */
 u8 *bitcoin_redeem_p2pkh(const tal_t *ctx, const struct pubkey *pubkey,
@@ -40,10 +38,6 @@ u8 *bitcoin_redeem_p2pkh(const tal_t *ctx, const struct pubkey *pubkey,
 /* Create the redeemscript for a P2SH + P2WPKH. */
 u8 *bitcoin_redeem_p2sh_p2wpkh(const tal_t *ctx, const struct pubkey *key);
 
-/* Create the scriptsig for a redeemscript */
-u8 *bitcoin_scriptsig_redeem(const tal_t *ctx,
-			     const u8 *redeemscript TAKES);
-
 /* Create scriptsig for p2sh-p2wpkh */
 u8 *bitcoin_scriptsig_p2sh_p2wpkh(const tal_t *ctx, const struct pubkey *key);
 
@@ -63,23 +57,9 @@ u8 *scriptpubkey_p2wpkh_derkey(const tal_t *ctx, const u8 der[33]);
 u8 *scriptpubkey_witness_raw(const tal_t *ctx, u8 version,
 			     const u8 *wprog, size_t wprog_size);
 
-/* Create an output script for a "raw"(perhaps already tweaked) taproot output pubkey */
-u8 *scriptpubkey_raw_p2tr(const tal_t *ctx, const struct pubkey *output_pubkey);
-
-/* Same as above, but compressed key is DER-encoded. */
-u8 *scriptpubkey_raw_p2tr_derkey(const tal_t *ctx, const u8 output_der[33]);
-
-/* Create an output script for an internal taproot pubkey. Results in different script than
- * scriptpubkey_raw_p2tr! TODO support merkle root tweaking */
-u8 *scriptpubkey_p2tr(const tal_t *ctx, const struct pubkey *inner_pubkey);
-
-/* Same as above, but compressed key is DER-encoded. TODO support merkle root tweaking */
-u8 *scriptpubkey_p2tr_derkey(const tal_t *ctx, const u8 inner_der[33]);
-
-/* To-remotekey with csv max(lease_expiry - blockheight, 1) delay. */
-u8 *bitcoin_wscript_to_remote_anchored(const tal_t *ctx,
-				       const struct pubkey *remote_key,
-				       u32 csv_lock);
+/* To-remotekey with csv 1 delay. */
+u8 *anchor_to_remote_redeem(const tal_t *ctx,
+			    const struct pubkey *remote_key);
 
 /* Create a witness which spends the 2of2. */
 u8 **bitcoin_witness_2of2(const tal_t *ctx,
@@ -100,8 +80,8 @@ u8 **bitcoin_witness_sig_and_element(const tal_t *ctx,
 				     const u8 *witnessscript);
 
 /* BOLT #3 to-local output */
-u8 *bitcoin_wscript_to_local(const tal_t *ctx, u16 to_self_delay,
-			     u32 lease_remaining,
+u8 *bitcoin_wscript_to_local(const tal_t *ctx,
+			     u16 to_self_delay,
 			     const struct pubkey *revocation_pubkey,
 			     const struct pubkey *local_delayedkey);
 
@@ -111,8 +91,7 @@ u8 *bitcoin_wscript_htlc_offer(const tal_t *ctx,
 			       const struct pubkey *remotehtlckey,
 			       const struct sha256 *payment_hash,
 			       const struct pubkey *revocationkey,
-			       bool option_anchor_outputs,
-			       bool option_anchors_zero_fee_htlc_tx);
+			       bool option_anchor_outputs);
 u8 **bitcoin_witness_htlc_timeout_tx(const tal_t *ctx,
 				     const struct bitcoin_signature *localsig,
 				     const struct bitcoin_signature *remotesig,
@@ -123,8 +102,7 @@ u8 *bitcoin_wscript_htlc_receive(const tal_t *ctx,
 				 const struct pubkey *remotekey,
 				 const struct sha256 *payment_hash,
 				 const struct pubkey *revocationkey,
-				 bool option_anchor_outputs,
-				 bool option_anchors_zero_fee_htlc_tx);
+				 bool option_anchor_outputs);
 u8 **bitcoin_witness_htlc_success_tx(const tal_t *ctx,
 				     const struct bitcoin_signature *localsig,
 				     const struct bitcoin_signature *remotesig,
@@ -137,16 +115,14 @@ u8 *bitcoin_wscript_htlc_offer_ripemd160(const tal_t *ctx,
 					 const struct pubkey *remotehtlckey,
 					 const struct ripemd160 *payment_ripemd,
 					 const struct pubkey *revocationkey,
-					 bool option_anchor_outputs,
-					 bool option_anchors_zero_fee_htlc_tx);
+					 bool option_anchor_outputs);
 u8 *bitcoin_wscript_htlc_receive_ripemd(const tal_t *ctx,
 					const struct abs_locktime *htlc_abstimeout,
 					const struct pubkey *localkey,
 					const struct pubkey *remotekey,
 					const struct ripemd160 *payment_ripemd,
 					const struct pubkey *revocationkey,
-					bool option_anchor_outputs,
-					bool option_anchors_zero_fee_htlc_tx);
+					bool option_anchor_outputs);
 
 /* BOLT #3 HTLC-success/HTLC-timeout output */
 u8 *bitcoin_wscript_htlc_tx(const tal_t *ctx,
@@ -159,28 +135,19 @@ u8 *bitcoin_wscript_anchor(const tal_t *ctx,
 			   const struct pubkey *funding_pubkey);
 
 /* Is this a pay to pubkey hash? (extract addr if not NULL) */
-bool is_p2pkh(const u8 *script, size_t script_len, struct bitcoin_address *addr);
+bool is_p2pkh(const u8 *script, struct bitcoin_address *addr);
 
 /* Is this a pay to script hash? (extract addr if not NULL) */
-bool is_p2sh(const u8 *script, size_t script_len, struct ripemd160 *addr);
+bool is_p2sh(const u8 *script, struct ripemd160 *addr);
 
 /* Is this (version 0) pay to witness script hash? (extract addr if not NULL) */
-bool is_p2wsh(const u8 *script, size_t script_len, struct sha256 *addr);
+bool is_p2wsh(const u8 *script, struct sha256 *addr);
 
 /* Is this (version 0) pay to witness pubkey hash? (extract addr if not NULL) */
-bool is_p2wpkh(const u8 *script, size_t script_len, struct bitcoin_address *addr);
+bool is_p2wpkh(const u8 *script, struct bitcoin_address *addr);
 
-/* Is this a taproot output? (extract xonly_pubkey bytes if not NULL) */
-bool is_p2tr(const u8 *script, size_t script_len, u8 xonly_pubkey[32]);
-
-/* Is this one of the above script types? */
-bool is_known_scripttype(const u8 *script, size_t script_len);
-
-/* Is this a witness script type? */
-bool is_known_segwit_scripttype(const u8 *script, size_t script_len);
-
-/* Is this a to-remote witness script (used for option_anchor_outputs)? */
-bool is_to_remote_anchored_witness_script(const u8 *script, size_t script_len);
+/* Is this one of the four above script types? */
+bool is_known_scripttype(const u8 *script);
 
 /* Is this an anchor witness script? */
 bool is_anchor_witness_script(const u8 *script, size_t script_len);
@@ -203,7 +170,4 @@ void script_push_bytes(u8 **scriptp, const void *mem, size_t len);
 /* OP_0 + PUSH(32-byte-hash) */
 #define BITCOIN_SCRIPTPUBKEY_P2WSH_LEN (1 + 1 + 32)
 
-/* OP_1 + PUSH(32-byte-key) */
-#define BITCOIN_SCRIPTPUBKEY_P2TR_LEN (1 + 1 + 32)
-
 #endif /* LIGHTNING_BITCOIN_SCRIPT_H */

bitcoin/shadouble.c

@@ -1,7 +1,6 @@
-#include "config.h"
-#include <bitcoin/shadouble.h>
+#include "shadouble.h"
 #include <ccan/mem/mem.h>
-#include <common/utils.h>
+#include <common/type_to_string.h>
 #include <wire/wire.h>
 
 void sha256_double(struct sha256_double *shadouble, const void *p, size_t len)
@@ -15,11 +14,7 @@ void sha256_double_done(struct sha256_ctx *shactx, struct sha256_double *res)
 	sha256_done(shactx, &res->sha);
 	sha256(&res->sha, &res->sha, sizeof(res->sha));
 }
-
-char *fmt_sha256_double(const tal_t *ctx, const struct sha256_double *shad)
-{
-	return tal_hexstr(ctx, shad, sizeof(*shad));
-}
+REGISTER_TYPE_TO_HEXSTR(sha256_double);
 
 void towire_sha256_double(u8 **pptr, const struct sha256_double *sha256d)
 {

bitcoin/shadouble.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/crypto/sha256/sha256.h>
 #include <ccan/short_types/short_types.h>
-#include <ccan/tal/tal.h>
 
 /* To explicitly distinguish between single sha and bitcoin's standard double */
 struct sha256_double {
@@ -18,6 +17,4 @@ void sha256_double_done(struct sha256_ctx *sha256, struct sha256_double *res);
 void fromwire_sha256_double(const u8 **cursor, size_t *max,
 			    struct sha256_double *sha256d);
 void towire_sha256_double(u8 **pptr, const struct sha256_double *sha256d);
-
-char *fmt_sha256_double(const tal_t *ctx, const struct sha256_double *shad);
 #endif /* LIGHTNING_BITCOIN_SHADOUBLE_H */

bitcoin/short_channel_id.c

@@ -1,7 +1,8 @@
-#include "config.h"
 #include <bitcoin/short_channel_id.h>
 #include <ccan/tal/str/str.h>
+#include <common/type_to_string.h>
 #include <stdio.h>
+#include <string.h>
 #include <wire/wire.h>
 
 /* BOLT#07:
@@ -52,7 +53,7 @@ bool short_channel_id_from_str(const char *str, size_t strlen,
 		&& mk_short_channel_id(dst, blocknum, txnum, outnum);
 }
 
-char *fmt_short_channel_id(const tal_t *ctx, struct short_channel_id scid)
+char *short_channel_id_to_str(const tal_t *ctx, const struct short_channel_id *scid)
 {
 	return tal_fmt(ctx, "%dx%dx%d",
 		       short_channel_id_blocknum(scid),
@@ -77,25 +78,40 @@ bool short_channel_id_dir_from_str(const char *str, size_t strlen,
 	return true;
 }
 
-char *fmt_short_channel_id_dir(const tal_t *ctx,
-			       const struct short_channel_id_dir *scidd)
+char *short_channel_id_dir_to_str(const tal_t *ctx,
+				  const struct short_channel_id_dir *scidd)
 {
-	char *str, *scidstr = fmt_short_channel_id(NULL, scidd->scid);
+	char *str, *scidstr = short_channel_id_to_str(NULL, &scidd->scid);
 	str = tal_fmt(ctx, "%s/%u", scidstr, scidd->dir);
 	tal_free(scidstr);
 	return str;
 }
 
+REGISTER_TYPE_TO_STRING(short_channel_id, short_channel_id_to_str);
+REGISTER_TYPE_TO_STRING(short_channel_id_dir, short_channel_id_dir_to_str);
+
 void towire_short_channel_id(u8 **pptr,
-			     struct short_channel_id short_channel_id)
+			     const struct short_channel_id *short_channel_id)
 {
-	towire_u64(pptr, short_channel_id.u64);
+	towire_u64(pptr, short_channel_id->u64);
 }
 
-struct short_channel_id fromwire_short_channel_id(const u8 **cursor, size_t *max)
+void towire_short_channel_id_dir(u8 **pptr,
+				 const struct short_channel_id_dir *scidd)
 {
-	struct short_channel_id scid;
-
-	scid.u64 = fromwire_u64(cursor, max);
-	return scid;
+	towire_short_channel_id(pptr, &scidd->scid);
+	towire_bool(pptr, scidd->dir);
+}
+
+void fromwire_short_channel_id(const u8 **cursor, size_t *max,
+			       struct short_channel_id *short_channel_id)
+{
+	short_channel_id->u64 = fromwire_u64(cursor, max);
+}
+
+void fromwire_short_channel_id_dir(const u8 **cursor, size_t *max,
+				   struct short_channel_id_dir *scidd)
+{
+	fromwire_short_channel_id(cursor, max, &scidd->scid);
+	scidd->dir = fromwire_bool(cursor, max);
 }

bitcoin/short_channel_id.h

@@ -1,28 +1,20 @@
 #ifndef LIGHTNING_BITCOIN_SHORT_CHANNEL_ID_H
 #define LIGHTNING_BITCOIN_SHORT_CHANNEL_ID_H
 #include "config.h"
-#include <ccan/compiler/compiler.h>
 #include <ccan/short_types/short_types.h>
+#include <ccan/structeq/structeq.h>
 #include <ccan/tal/tal.h>
 #include <common/gossip_constants.h>
+#include <stdbool.h>
+#include <stddef.h>
 
 /* Short Channel ID is composed of 3 bytes for the block height, 3
  * bytes of tx index in block and 2 bytes of output index. */
 struct short_channel_id {
 	u64 u64;
 };
-
-static inline bool short_channel_id_eq(struct short_channel_id a,
-				       struct short_channel_id b)
-{
-	return a.u64 == b.u64;
-}
-
-static inline size_t short_channel_id_hash(struct short_channel_id scid)
-{
-	/* scids cost money to generate, so simple hash works here */
-	return (scid.u64 >> 32) ^ (scid.u64 >> 16) ^ scid.u64;
-}
+/* Define short_channel_id_eq (no padding) */
+STRUCTEQ_DEF(short_channel_id, 0, u64);
 
 /* BOLT #7:
  *
@@ -41,38 +33,25 @@ struct short_channel_id_dir {
 	int dir;
 };
 
-static inline bool short_channel_id_dir_eq(const struct short_channel_id_dir *a,
-					   const struct short_channel_id_dir *b)
+static inline u32 short_channel_id_blocknum(const struct short_channel_id *scid)
 {
-	return short_channel_id_eq(a->scid, b->scid) && a->dir == b->dir;
+	return scid->u64 >> 40;
 }
 
-static inline u32 short_channel_id_blocknum(struct short_channel_id scid)
+static inline u32 short_channel_id_txnum(const struct short_channel_id *scid)
 {
-	return scid.u64 >> 40;
+	return (scid->u64 >> 16) & 0x00FFFFFF;
 }
 
-static inline bool is_stub_scid(struct short_channel_id scid)
+static inline u16 short_channel_id_outnum(const struct short_channel_id *scid)
 {
-	return scid.u64 >> 40 == 1 &&
-		((scid.u64 >> 16) & 0x00FFFFFF) == 1 &&
-		(scid.u64 & 0xFFFF) == 1;
-}
-
-static inline u32 short_channel_id_txnum(struct short_channel_id scid)
-{
-	return (scid.u64 >> 16) & 0x00FFFFFF;
-}
-
-static inline u16 short_channel_id_outnum(struct short_channel_id scid)
-{
-	return scid.u64 & 0xFFFF;
+	return scid->u64 & 0xFFFF;
 }
 
 /* Subtly, at block N, depth is 1, hence the -1 here. eg. 103x1x0 is announceable
  * when height is 108. */
 static inline bool
-is_scid_depth_announceable(struct short_channel_id scid,
+is_scid_depth_announceable(const struct short_channel_id *scid,
 			  unsigned int height)
 {
 	return short_channel_id_blocknum(scid) + ANNOUNCE_MIN_DEPTH - 1
@@ -86,16 +65,22 @@ bool WARN_UNUSED_RESULT mk_short_channel_id(struct short_channel_id *scid,
 bool WARN_UNUSED_RESULT short_channel_id_from_str(const char *str, size_t strlen,
 						  struct short_channel_id *dst);
 
+char *short_channel_id_to_str(const tal_t *ctx, const struct short_channel_id *scid);
+
 bool WARN_UNUSED_RESULT short_channel_id_dir_from_str(const char *str, size_t strlen,
 						      struct short_channel_id_dir *scidd);
 
-char *fmt_short_channel_id(const tal_t *ctx, struct short_channel_id scid);
-char *fmt_short_channel_id_dir(const tal_t *ctx,
-			       const struct short_channel_id_dir *scidd);
+char *short_channel_id_dir_to_str(const tal_t *ctx,
+				  const struct short_channel_id_dir *scidd);
 
 /* Marshal/unmarshal */
 void towire_short_channel_id(u8 **pptr,
-			     struct short_channel_id short_channel_id);
-struct short_channel_id fromwire_short_channel_id(const u8 **cursor, size_t *max);
+			     const struct short_channel_id *short_channel_id);
+void towire_short_channel_id_dir(u8 **pptr,
+				 const struct short_channel_id_dir *scidd);
+void fromwire_short_channel_id(const u8 **cursor, size_t *max,
+			       struct short_channel_id *short_channel_id);
+void fromwire_short_channel_id_dir(const u8 **cursor, size_t *max,
+				   struct short_channel_id_dir *scidd);
 
 #endif /* LIGHTNING_BITCOIN_SHORT_CHANNEL_ID_H */

bitcoin/signature.c

@@ -1,15 +1,15 @@
-#include "config.h"
+#include "privkey.h"
+#include "pubkey.h"
+#include "script.h"
+#include "shadouble.h"
+#include "signature.h"
+#include "tx.h"
 #include <assert.h>
-#include <bitcoin/privkey.h>
 #include <bitcoin/psbt.h>
-#include <bitcoin/pubkey.h>
-#include <bitcoin/script.h>
-#include <bitcoin/shadouble.h>
-#include <bitcoin/signature.h>
-#include <bitcoin/tx.h>
+#include <ccan/cast/cast.h>
 #include <ccan/mem/mem.h>
+#include <common/type_to_string.h>
 #include <common/utils.h>
-#include <secp256k1_schnorrsig.h>
 #include <wire/wire.h>
 
 #undef DEBUG
@@ -94,8 +94,6 @@ static bool sig_has_low_r(const secp256k1_ecdsa_signature* sig)
 	return compact_sig[0] < 0x80;
 }
 
-bool dev_no_signature_grind = false;
-
 void sign_hash(const struct privkey *privkey,
 	       const struct sha256_double *h,
 	       secp256k1_ecdsa_signature *s)
@@ -108,12 +106,8 @@ void sign_hash(const struct privkey *privkey,
 		ok = secp256k1_ecdsa_sign(secp256k1_ctx,
 					  s,
 					  h->sha.u.u8,
-					  privkey->secret.data, NULL,
-					  dev_no_signature_grind ? NULL
-					  : extra_entropy);
+					  privkey->secret.data, NULL, extra_entropy);
 		((u32 *)extra_entropy)[0]++;
-		if (dev_no_signature_grind)
-			break;
 	} while (!sig_has_low_r(s));
 
 	assert(ok);
@@ -133,8 +127,6 @@ void bitcoin_tx_hash_for_sig(const struct bitcoin_tx *tx, unsigned int in,
 	input_amt = psbt_input_get_amount(tx->psbt, in);
 	input_val_sats = input_amt.satoshis; /* Raw: type conversion */
 
-	/* Wally can allocate here, iff tx doesn't fit on stack */
-	tal_wally_start();
 	if (is_elements(chainparams)) {
 		ret = wally_tx_confidential_value_from_satoshi(input_val_sats, value, sizeof(value));
 		assert(ret == WALLY_OK);
@@ -149,7 +141,6 @@ void bitcoin_tx_hash_for_sig(const struct bitcoin_tx *tx, unsigned int in,
 		    sighash_type, flags, dest->sha.u.u8, sizeof(*dest));
 		assert(ret == WALLY_OK);
 	}
-	tal_wally_end(tx->wtx);
 }
 
 void sign_tx_input(const struct bitcoin_tx *tx,
@@ -179,14 +170,6 @@ bool check_signed_hash(const struct sha256_double *hash,
 {
 	int ret;
 
-	/* BOLT #2:
-	 *
-	 * - if `signature` is incorrect OR non-compliant with
-	 *   LOW-S-standard rule
-	 */
-	/* From the secp256k1_ecdsa_verify documentation: "To avoid
-	 * accepting malleable signatures, only ECDSA signatures in
-	 * lower-S form are accepted." */
 	ret = secp256k1_ecdsa_verify(secp256k1_ctx,
 				     signature,
 				     hash->sha.u.u8, &key->pubkey);
@@ -213,8 +196,8 @@ bool check_tx_sig(const struct bitcoin_tx *tx, size_t input_num,
 	}
 	assert(input_num < tx->wtx->num_inputs);
 
-	bitcoin_tx_hash_for_sig(tx, input_num, script, sig->sighash_type, &hash);
 	dump_tx("check_tx_sig", tx, input_num, script, key, &hash);
+	bitcoin_tx_hash_for_sig(tx, input_num, script, sig->sighash_type, &hash);
 
 	ret = check_signed_hash(&hash, &sig->s, key);
 	if (!ret)
@@ -324,7 +307,8 @@ bool signature_from_der(const u8 *der, size_t len, struct bitcoin_signature *sig
 	return true;
 }
 
-char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx, const secp256k1_ecdsa_signature *sig)
+static char *signature_to_hexstr(const tal_t *ctx,
+				 const secp256k1_ecdsa_signature *sig)
 {
 	u8 der[72];
 	size_t len = 72;
@@ -334,15 +318,17 @@ char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx, const secp256k1_ecdsa_sign
 
 	return tal_hexstr(ctx, der, len);
 }
+REGISTER_TYPE_TO_STRING(secp256k1_ecdsa_signature, signature_to_hexstr);
 
-char *fmt_bitcoin_signature(const tal_t *ctx,
-			    const struct bitcoin_signature *sig)
+static char *bitcoin_signature_to_hexstr(const tal_t *ctx,
+					 const struct bitcoin_signature *sig)
 {
 	u8 der[73];
 	size_t len = signature_to_der(der, sig);
 
 	return tal_hexstr(ctx, der, len);
 }
+REGISTER_TYPE_TO_STRING(bitcoin_signature, bitcoin_signature_to_hexstr);
 
 void fromwire_bitcoin_signature(const u8 **cursor, size_t *max,
 				struct bitcoin_signature *sig)
@@ -359,74 +345,3 @@ void towire_bitcoin_signature(u8 **pptr, const struct bitcoin_signature *sig)
 	towire_secp256k1_ecdsa_signature(pptr, &sig->s);
 	towire_u8(pptr, sig->sighash_type);
 }
-
-void towire_bip340sig(u8 **pptr, const struct bip340sig *bip340sig)
-{
-	towire_u8_array(pptr, bip340sig->u8, sizeof(bip340sig->u8));
-}
-
-void fromwire_bip340sig(const u8 **cursor, size_t *max,
-			struct bip340sig *bip340sig)
-{
-	fromwire_u8_array(cursor, max, bip340sig->u8, sizeof(bip340sig->u8));
-}
-
-char *fmt_bip340sig(const tal_t *ctx, const struct bip340sig *bip340sig)
-{
-	return tal_hexstr(ctx, bip340sig->u8, sizeof(bip340sig->u8));
-}
-
-/* BIP-340:
- *
- * This proposal suggests to include the tag by prefixing the hashed
- * data with ''SHA256(tag) || SHA256(tag)''. Because this is a 64-byte
- * long context-specific constant and the ''SHA256'' block size is
- * also 64 bytes, optimized implementations are possible (identical to
- * SHA256 itself, but with a modified initial state). Using SHA256 of
- * the tag name itself is reasonably simple and efficient for
- * implementations that don't choose to use the optimization.
- */
-
-/* For caller convenience, we hand in tag in parts (any can be "") */
-void bip340_sighash_init(struct sha256_ctx *sctx,
-			 const char *tag1,
-			 const char *tag2,
-			 const char *tag3)
-{
-	struct sha256 taghash;
-
-	sha256_init(sctx);
-	sha256_update(sctx, memcheck(tag1, strlen(tag1)), strlen(tag1));
-	sha256_update(sctx, memcheck(tag2, strlen(tag2)), strlen(tag2));
-	sha256_update(sctx, memcheck(tag3, strlen(tag3)), strlen(tag3));
-	sha256_done(sctx, &taghash);
-
-	sha256_init(sctx);
-	sha256_update(sctx, &taghash, sizeof(taghash));
-	sha256_update(sctx, &taghash, sizeof(taghash));
-}
-
-
-bool check_schnorr_sig(const struct sha256 *hash,
-		       const secp256k1_pubkey *pubkey,
-		       const struct bip340sig *sig)
-{
-	/* FIXME: uuuugly!  There's no non-xonly verify function. */
-	u8 raw[PUBKEY_CMPR_LEN];
-	size_t outlen = sizeof(raw);
-	secp256k1_xonly_pubkey xonly_pubkey;
-
-	if (!secp256k1_ec_pubkey_serialize(secp256k1_ctx, raw, &outlen,
-					   pubkey,
-					   SECP256K1_EC_COMPRESSED))
-		abort();
-	assert(outlen == PUBKEY_CMPR_LEN);
-	if (!secp256k1_xonly_pubkey_parse(secp256k1_ctx, &xonly_pubkey, raw+1))
-		abort();
-
- 	return secp256k1_schnorrsig_verify(secp256k1_ctx,
- 					   sig->u8,
- 					   hash->u.u8,
- 					   sizeof(hash->u.u8),
-					   &xonly_pubkey) == 1;
-}

bitcoin/signature.h

@@ -2,17 +2,14 @@
 #define LIGHTNING_BITCOIN_SIGNATURE_H
 #include "config.h"
 #include <ccan/short_types/short_types.h>
-#include <ccan/tal/tal.h>
 #include <secp256k1.h>
+#include <stdbool.h>
 
-struct sha256;
 struct sha256_double;
-struct sha256_ctx;
 struct bitcoin_tx;
 struct pubkey;
 struct privkey;
 struct bitcoin_tx_output;
-struct bip340sig;
 
 enum sighash_type {
     SIGHASH_ALL = 1,
@@ -123,13 +120,6 @@ bool check_tx_sig(const struct bitcoin_tx *tx, size_t input_num,
 		  const struct pubkey *key,
 		  const struct bitcoin_signature *sig);
 
-/**
- * check a Schnorr signature
- */
-bool check_schnorr_sig(const struct sha256 *hash,
-		       const secp256k1_pubkey *pubkey,
-		       const struct bip340sig *sig);
-
 /* Give DER encoding of signature: returns length used (<= 73). */
 size_t signature_to_der(u8 der[73], const struct bitcoin_signature *sig);
 
@@ -141,28 +131,4 @@ void towire_bitcoin_signature(u8 **pptr, const struct bitcoin_signature *sig);
 void fromwire_bitcoin_signature(const u8 **cursor, size_t *max,
 				struct bitcoin_signature *sig);
 
-/* Schnorr */
-struct bip340sig {
-	u8 u8[64];
-};
-void towire_bip340sig(u8 **pptr, const struct bip340sig *bip340sig);
-void fromwire_bip340sig(const u8 **cursor, size_t *max,
-			struct bip340sig *bip340sig);
-
-/* Get a hex string sig */
-char *fmt_secp256k1_ecdsa_signature(const tal_t *ctx,
-				    const secp256k1_ecdsa_signature *sig);
-char *fmt_bip340sig(const tal_t *ctx, const struct bip340sig *bip340sig);
-char *fmt_bitcoin_signature(const tal_t *ctx,
-			    const struct bitcoin_signature *sig);
-
-/* For caller convenience, we hand in tag in parts (any can be "") */
-void bip340_sighash_init(struct sha256_ctx *sctx,
-			 const char *tag1,
-			 const char *tag2,
-			 const char *tag3);
-
-/* Some of the spec test vectors assume no sig grinding. */
-extern bool dev_no_signature_grind;
-
 #endif /* LIGHTNING_BITCOIN_SIGNATURE_H */

bitcoin/test/Makefile

@@ -2,15 +2,16 @@ BITCOIN_TEST_SRC := $(wildcard bitcoin/test/run-*.c)
 BITCOIN_TEST_OBJS := $(BITCOIN_TEST_SRC:.c=.o)
 BITCOIN_TEST_PROGRAMS := $(BITCOIN_TEST_OBJS:.o=)
 
-BITCOIN_TEST_COMMON_OBJS := common/utils.o common/setup.o common/autodata.o
+BITCOIN_TEST_COMMON_OBJS := common/utils.o common/setup.o
 
-$(BITCOIN_TEST_PROGRAMS): $(BITCOIN_TEST_COMMON_OBJS) bitcoin/chainparams.o
+$(BITCOIN_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_TEST_COMMON_OBJS) bitcoin/chainparams.o
 $(BITCOIN_TEST_OBJS): $(CCAN_HEADERS) $(BITCOIN_HEADERS) $(BITCOIN_SRC)
 
 ALL_TEST_PROGRAMS += $(BITCOIN_TEST_PROGRAMS)
-ALL_C_SOURCES += $(BITCOIN_TEST_PROGRAMS:=.c)
+ALL_OBJS += $(BITCOIN_TEST_PROGRAMS:=.o)
 
 # This needs to know what level of optimization we're using.
 bitcoin/test/run-secret_eq_consttime.o: CFLAGS += -DCOPTFLAGS="\"${COPTFLAGS}\""
+update-mocks: $(BITCOIN_TEST_SRC:%=update-mocks/%)
 
 check-units: $(BITCOIN_TEST_PROGRAMS:%=unittest/%)

bitcoin/test/run-bitcoin_block_from_hex.c

@@ -1,24 +1,21 @@
-#include "config.h"
 #include "../block.c"
 #include "../psbt.c"
 #include "../shadouble.c"
-#include "../signature.c"
 #include "../tx.c"
 #include "../varint.c"
 #include <assert.h>
 #include <common/setup.h>
-#include <stdio.h>
 
 /* AUTOGENERATED MOCKS START */
+/* Generated stub for amount_asset_extract_value */
+u8 *amount_asset_extract_value(const tal_t *ctx UNNEEDED, struct amount_asset *asset UNNEEDED)
+{ fprintf(stderr, "amount_asset_extract_value called!\n"); abort(); }
 /* Generated stub for amount_asset_is_main */
 bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_is_main called!\n"); abort(); }
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
 /* Generated stub for amount_sat */
 struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 { fprintf(stderr, "amount_sat called!\n"); abort(); }
@@ -47,53 +44,61 @@ struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED)
 /* Generated stub for fromwire */
 const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED)
 { fprintf(stderr, "fromwire called!\n"); abort(); }
+/* Generated stub for fromwire_amount_sat */
+struct amount_sat fromwire_amount_sat(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_amount_sat called!\n"); abort(); }
 /* Generated stub for fromwire_fail */
 void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_fail called!\n"); abort(); }
-/* Generated stub for fromwire_secp256k1_ecdsa_signature */
-void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-					secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); }
 /* Generated stub for fromwire_sha256 */
 void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED)
 { fprintf(stderr, "fromwire_sha256 called!\n"); abort(); }
+/* Generated stub for fromwire_tal_arrn */
+u8 *fromwire_tal_arrn(const tal_t *ctx UNNEEDED,
+		       const u8 **cursor UNNEEDED, size_t *max UNNEEDED, size_t num UNNEEDED)
+{ fprintf(stderr, "fromwire_tal_arrn called!\n"); abort(); }
+/* Generated stub for fromwire_u16 */
+u16 fromwire_u16(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_u16 called!\n"); abort(); }
 /* Generated stub for fromwire_u32 */
 u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_u32 called!\n"); abort(); }
-/* Generated stub for fromwire_u8 */
-u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); }
-/* Generated stub for fromwire_u8_array */
-void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED)
-{ fprintf(stderr, "fromwire_u8_array called!\n"); abort(); }
 /* Generated stub for is_anchor_witness_script */
 bool is_anchor_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
 { fprintf(stderr, "is_anchor_witness_script called!\n"); abort(); }
-/* Generated stub for is_to_remote_anchored_witness_script */
-bool is_to_remote_anchored_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_to_remote_anchored_witness_script called!\n"); abort(); }
+/* Generated stub for is_p2sh */
+bool is_p2sh(const u8 *script UNNEEDED, struct ripemd160 *addr UNNEEDED)
+{ fprintf(stderr, "is_p2sh called!\n"); abort(); }
+/* Generated stub for is_p2wpkh */
+bool is_p2wpkh(const u8 *script UNNEEDED, struct bitcoin_address *addr UNNEEDED)
+{ fprintf(stderr, "is_p2wpkh called!\n"); abort(); }
+/* Generated stub for is_p2wsh */
+bool is_p2wsh(const u8 *script UNNEEDED, struct sha256 *addr UNNEEDED)
+{ fprintf(stderr, "is_p2wsh called!\n"); abort(); }
 /* Generated stub for pubkey_to_der */
 void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UNNEEDED)
 { fprintf(stderr, "pubkey_to_der called!\n"); abort(); }
 /* Generated stub for pubkey_to_hash160 */
 void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
 { fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
+/* Generated stub for script_push_bytes */
+void script_push_bytes(u8 **scriptp UNNEEDED, const void *mem UNNEEDED, size_t len UNNEEDED)
+{ fprintf(stderr, "script_push_bytes called!\n"); abort(); }
 /* Generated stub for scriptpubkey_p2wsh */
 u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
 { fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }
-/* Generated stub for towire_secp256k1_ecdsa_signature */
-void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED,
-			      const secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); }
+/* Generated stub for towire_amount_sat */
+void towire_amount_sat(u8 **pptr UNNEEDED, const struct amount_sat sat UNNEEDED)
+{ fprintf(stderr, "towire_amount_sat called!\n"); abort(); }
 /* Generated stub for towire_sha256 */
 void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED)
 { fprintf(stderr, "towire_sha256 called!\n"); abort(); }
+/* Generated stub for towire_u16 */
+void towire_u16(u8 **pptr UNNEEDED, u16 v UNNEEDED)
+{ fprintf(stderr, "towire_u16 called!\n"); abort(); }
 /* Generated stub for towire_u32 */
 void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED)
 { fprintf(stderr, "towire_u32 called!\n"); abort(); }
-/* Generated stub for towire_u8 */
-void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED)
-{ fprintf(stderr, "towire_u8 called!\n"); abort(); }
 /* Generated stub for towire_u8_array */
 void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED)
 { fprintf(stderr, "towire_u8_array called!\n"); abort(); }
@@ -149,15 +154,6 @@ static const char block[] =
 
 STRUCTEQ_DEF(sha256_double, 0, sha);
 
-static bool bitcoin_blkid_from_hex(const char *hexstr, size_t hexstr_len,
-				   struct bitcoin_blkid *blockid)
-{
-	struct bitcoin_txid fake_txid;
-	if (!bitcoin_txid_from_hex(hexstr, hexstr_len, &fake_txid))
-		return false;
-	blockid->shad = fake_txid.shad;
-	return true;
-}
 int main(int argc, const char *argv[])
 {
 	struct bitcoin_blkid prev;
@@ -171,13 +167,13 @@ int main(int argc, const char *argv[])
 				   block, strlen(block));
 
 	assert(b);
-	assert(b->hdr.version == 0x6592a000);
+	assert(b->hdr.version == CPU_TO_LE32(0x6592a000));
 	bitcoin_blkid_from_hex("0000000000000f31173e973bc00e452b1fac350066df7db2adec1e3224ea5bc1", strlen("0000000000000f31173e973bc00e452b1fac350066df7db2adec1e3224ea5bc1"), &prev);
 	assert(bitcoin_blkid_eq(&prev, &b->hdr.prev_hash));
 	hex_decode("8a0ee58ded5de949325ebc99583e3ca84f96a6597465c611685413f50f0ead7e", strlen("8a0ee58ded5de949325ebc99583e3ca84f96a6597465c611685413f50f0ead7e"), &merkle, sizeof(merkle));
 	assert(sha256_double_eq(&merkle, &b->hdr.merkle_hash));
-	assert(b->hdr.timestamp == 1550507183);
-	assert(b->hdr.nonce == 1226407989);
+	assert(b->hdr.timestamp == CPU_TO_LE32(1550507183));
+	assert(b->hdr.nonce == CPU_TO_LE32(1226407989));
 
 	assert(tal_count(b->tx) == 3);
 	bitcoin_txid(b->tx[0], &txid);

bitcoin/test/run-psbt-from-tx.c

@@ -1,121 +0,0 @@
-#include "config.h"
-#include "../psbt.c"
-#include "../tx.c"
-#include "../../wire/towire.c"
-#include "../../wire/fromwire.c"
-#include <assert.h>
-#include <ccan/cast/cast.h>
-#include <common/setup.h>
-#include <stdio.h>
-
-/* AUTOGENERATED MOCKS START */
-/* Generated stub for amount_asset_is_main */
-bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
-{ fprintf(stderr, "amount_asset_is_main called!\n"); abort(); }
-/* Generated stub for amount_asset_to_sat */
-struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
-{ fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
-/* Generated stub for amount_sat */
-struct amount_sat amount_sat(u64 satoshis UNNEEDED)
-{ fprintf(stderr, "amount_sat called!\n"); abort(); }
-/* Generated stub for amount_sat_add */
- bool amount_sat_add(struct amount_sat *val UNNEEDED,
-				       struct amount_sat a UNNEEDED,
-				       struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_add called!\n"); abort(); }
-/* Generated stub for amount_sat_eq */
-bool amount_sat_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_eq called!\n"); abort(); }
-/* Generated stub for amount_sat_greater_eq */
-bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); }
-/* Generated stub for amount_sat_sub */
- bool amount_sat_sub(struct amount_sat *val UNNEEDED,
-				       struct amount_sat a UNNEEDED,
-				       struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_sub called!\n"); abort(); }
-/* Generated stub for amount_sat_to_asset */
-struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u8 *asset UNNEEDED)
-{ fprintf(stderr, "amount_sat_to_asset called!\n"); abort(); }
-/* Generated stub for amount_tx_fee */
-struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_tx_fee called!\n"); abort(); }
-/* Generated stub for fromwire_sha256_double */
-void fromwire_sha256_double(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-			    struct sha256_double *sha256d UNNEEDED)
-{ fprintf(stderr, "fromwire_sha256_double called!\n"); abort(); }
-/* Generated stub for is_anchor_witness_script */
-bool is_anchor_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_anchor_witness_script called!\n"); abort(); }
-/* Generated stub for is_to_remote_anchored_witness_script */
-bool is_to_remote_anchored_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_to_remote_anchored_witness_script called!\n"); abort(); }
-/* Generated stub for pubkey_to_der */
-void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UNNEEDED)
-{ fprintf(stderr, "pubkey_to_der called!\n"); abort(); }
-/* Generated stub for pubkey_to_hash160 */
-void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
-{ fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
-/* Generated stub for scriptpubkey_p2wsh */
-u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
-{ fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }
-/* Generated stub for sha256_double */
-void sha256_double(struct sha256_double *shadouble UNNEEDED, const void *p UNNEEDED, size_t len UNNEEDED)
-{ fprintf(stderr, "sha256_double called!\n"); abort(); }
-/* Generated stub for signature_from_der */
-bool signature_from_der(const u8 *der UNNEEDED, size_t len UNNEEDED, struct bitcoin_signature *sig UNNEEDED)
-{ fprintf(stderr, "signature_from_der called!\n"); abort(); }
-/* Generated stub for signature_to_der */
-size_t signature_to_der(u8 der[73] UNNEEDED, const struct bitcoin_signature *sig UNNEEDED)
-{ fprintf(stderr, "signature_to_der called!\n"); abort(); }
-/* Generated stub for towire_sha256_double */
-void towire_sha256_double(u8 **pptr UNNEEDED, const struct sha256_double *sha256d UNNEEDED)
-{ fprintf(stderr, "towire_sha256_double called!\n"); abort(); }
-/* Generated stub for varint_put */
-size_t varint_put(u8 buf[VARINT_MAX_LEN] UNNEEDED, varint_t v UNNEEDED)
-{ fprintf(stderr, "varint_put called!\n"); abort(); }
-/* Generated stub for varint_size */
-size_t varint_size(varint_t v UNNEEDED)
-{ fprintf(stderr, "varint_size called!\n"); abort(); }
-/* AUTOGENERATED MOCKS END */
-
-/* This transaction has scriptSig data in it.
- * We expect that creating a new psbt from it will correctly
- * populate the PSBT object */
-static const char *raw_tx = "0200000000010151d12aa54cc6e59a6a92325a8315e93361d9805115a13aa5ba8dbcf30ffd858c000000001716001401fad90abcd66697e2592164722de4a95ebee165fdffffff02603c250200000000160014c2ccab171c2a5be9dab52ec41b825863024c546600093d00000000002200205b8cd3b914cf67cdd8fa6273c930353dd36476734fbd962102c2df53b90880cd02473044022001e73b1745d775521c758e70549ad79b1d076efc34303f416e66ff630f6088e402207b0aa44b35329ae4733463bc9f6ca433c5595f00a902a21c941945a24f8aa577012103d745445c9362665f22e0d96e9e766f273f3260dea39c8a76bfa05dd2684ddccf66000000";
-
-int main(int argc, char *argv[])
-{
-	struct bitcoin_tx *tx, *tx2;
-	u8 *msg;
-	size_t len;
-
-	common_setup(argv[0]);
-	chainparams = chainparams_for_network("bitcoin");
-
-	msg = tal_arr(tmpctx, u8, 0);
-	tx = bitcoin_tx_from_hex(tmpctx, raw_tx, strlen(raw_tx));
-
-	/* convert to wire format */
-	towire_bitcoin_tx(&msg, tx);
-
-	len = tal_bytelen(msg);
-	assert(len > 0);
-
-	tx2 = fromwire_bitcoin_tx(tmpctx,
-				  cast_const2(const u8 **, &msg), &len);
-	assert(tx2 != NULL);
-
-	/* Witness/scriptsig data is saved down into psbt */
-	assert(tx2->psbt->num_inputs == 1);
-	const struct wally_map_item *final_scriptsig = wally_map_get_integer(&tx2->psbt->inputs[0].psbt_fields, /* PSBT_IN_FINAL_SCRIPTSIG */ 0x07);
-	assert(final_scriptsig->value_len > 0);
-	assert(tx2->psbt->inputs[0].final_witness != NULL);
-	audit_psbt(tx2->psbt, tx2->psbt);
-
-	common_shutdown();
-	return 0;
-}

bitcoin/test/run-secret_eq_consttime.c

@@ -1,9 +1,7 @@
-#include "config.h"
 #include <assert.h>
 #include <bitcoin/privkey.c>
 #include <ccan/err/err.h>
 #include <ccan/time/time.h>
-#include <common/setup.h>
 #include <stdio.h>
 #include <unistd.h>
 
@@ -108,22 +106,17 @@ static bool secret_time_test(struct timerel (*test)(struct secret *s1,
 
 #define ITERATIONS 1000
 
-int main(int argc, char *argv[])
+int main(void)
 {
 	const char *v;
 	int const_success, nonconst_success = ITERATIONS, i;
-
-	common_setup(argv[0]);
+	double load;
+	setup_locale();
 
 	/* no point running this under valgrind. */
 	v = getenv("VALGRIND");
 	if (v && atoi(v) == 1)
-		goto exit;
-
-	/* this sometimes trips under CI */
-	v = getenv("SLOW_MACHINE");
-	if (v && atoi(v) == 1)
-		goto exit;
+		exit(0);
 
 	s1 = calloc(RUNS, sizeof(*s1));
 	s2 = calloc(RUNS, sizeof(*s2));
@@ -148,16 +141,17 @@ int main(int argc, char *argv[])
 	}
 
 	/* Now, check loadavg: if we weren't alone, that could explain results */
+	getloadavg(&load, 1);
+	if (load > 1.0)
+		errx(0, "Load %.2f: too high, ignoring", load);
+
 	if (const_success < ITERATIONS / 2)
 		errx(1, "Only const time %u/%u?", const_success, i);
 
 	if (nonconst_success < ITERATIONS / 2)
-		errx(1, "memcmp seemed const time %u/%u?",
-		     nonconst_success, i);
+		errx(1, "memcmp seemed const time %u/%u?", nonconst_success, i);
 	free(s1);
 	free(s2);
 
-exit:
-	common_shutdown();
 	return 0;
 }

bitcoin/test/run-tx-bitcoin_tx_2of2_input_witness_weight.c

@@ -1,164 +0,0 @@
-#include "config.h"
-#include <assert.h>
-#include <bitcoin/privkey.c>
-#include <bitcoin/pubkey.c>
-#include <bitcoin/script.c>
-#include <bitcoin/shadouble.c>
-#include <bitcoin/signature.c>
-#include <bitcoin/tx.c>
-#include <bitcoin/varint.c>
-#include <ccan/str/hex/hex.h>
-#include <common/setup.h>
-#include <common/utils.h>
-#include <stdio.h>
-
-/* AUTOGENERATED MOCKS START */
-/* Generated stub for amount_asset_is_main */
-bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
-{ fprintf(stderr, "amount_asset_is_main called!\n"); abort(); }
-/* Generated stub for amount_asset_to_sat */
-struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
-{ fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
-/* Generated stub for amount_sat_add */
- bool amount_sat_add(struct amount_sat *val UNNEEDED,
-				       struct amount_sat a UNNEEDED,
-				       struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_add called!\n"); abort(); }
-/* Generated stub for amount_sat_greater_eq */
-bool amount_sat_greater_eq(struct amount_sat a UNNEEDED, struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_greater_eq called!\n"); abort(); }
-/* Generated stub for amount_sat_sub */
- bool amount_sat_sub(struct amount_sat *val UNNEEDED,
-				       struct amount_sat a UNNEEDED,
-				       struct amount_sat b UNNEEDED)
-{ fprintf(stderr, "amount_sat_sub called!\n"); abort(); }
-/* Generated stub for amount_sat_to_asset */
-struct amount_asset amount_sat_to_asset(struct amount_sat *sat UNNEEDED, const u8 *asset UNNEEDED)
-{ fprintf(stderr, "amount_sat_to_asset called!\n"); abort(); }
-/* Generated stub for amount_tx_fee */
-struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_tx_fee called!\n"); abort(); }
-/* Generated stub for clone_psbt */
-struct wally_psbt *clone_psbt(const tal_t *ctx UNNEEDED, const struct wally_psbt *psbt UNNEEDED)
-{ fprintf(stderr, "clone_psbt called!\n"); abort(); }
-/* Generated stub for fromwire */
-const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED)
-{ fprintf(stderr, "fromwire called!\n"); abort(); }
-/* Generated stub for fromwire_fail */
-void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_fail called!\n"); abort(); }
-/* Generated stub for fromwire_secp256k1_ecdsa_signature */
-void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-					secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); }
-/* Generated stub for fromwire_sha256 */
-void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED)
-{ fprintf(stderr, "fromwire_sha256 called!\n"); abort(); }
-/* Generated stub for fromwire_u32 */
-u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_u32 called!\n"); abort(); }
-/* Generated stub for fromwire_u8 */
-u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); }
-/* Generated stub for fromwire_u8_array */
-void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED)
-{ fprintf(stderr, "fromwire_u8_array called!\n"); abort(); }
-/* Generated stub for fromwire_wally_psbt */
-struct wally_psbt *fromwire_wally_psbt(const tal_t *ctx UNNEEDED,
-				       const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_wally_psbt called!\n"); abort(); }
-/* Generated stub for new_psbt */
-struct wally_psbt *new_psbt(const tal_t *ctx UNNEEDED,
-			    const struct wally_tx *wtx UNNEEDED)
-{ fprintf(stderr, "new_psbt called!\n"); abort(); }
-/* Generated stub for psbt_add_output */
-struct wally_psbt_output *psbt_add_output(struct wally_psbt *psbt UNNEEDED,
-					  struct wally_tx_output *output UNNEEDED,
-					  size_t insert_at UNNEEDED)
-{ fprintf(stderr, "psbt_add_output called!\n"); abort(); }
-/* Generated stub for psbt_append_input */
-struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt UNNEEDED,
-					   const struct bitcoin_outpoint *outpoint UNNEEDED,
-					   u32 sequence UNNEEDED,
-					   const u8 *scriptSig UNNEEDED,
-					   const u8 *input_wscript UNNEEDED,
-					   const u8 *redeemscript UNNEEDED)
-{ fprintf(stderr, "psbt_append_input called!\n"); abort(); }
-/* Generated stub for psbt_final_tx */
-struct wally_tx *psbt_final_tx(const tal_t *ctx UNNEEDED, const struct wally_psbt *psbt UNNEEDED)
-{ fprintf(stderr, "psbt_final_tx called!\n"); abort(); }
-/* Generated stub for psbt_finalize */
-bool psbt_finalize(struct wally_psbt *psbt UNNEEDED)
-{ fprintf(stderr, "psbt_finalize called!\n"); abort(); }
-/* Generated stub for psbt_input_get_amount */
-struct amount_sat psbt_input_get_amount(const struct wally_psbt *psbt UNNEEDED,
-					size_t in UNNEEDED)
-{ fprintf(stderr, "psbt_input_get_amount called!\n"); abort(); }
-/* Generated stub for psbt_input_set_wit_utxo */
-void psbt_input_set_wit_utxo(struct wally_psbt *psbt UNNEEDED, size_t in UNNEEDED,
-			     const u8 *scriptPubkey UNNEEDED, struct amount_sat amt UNNEEDED)
-{ fprintf(stderr, "psbt_input_set_wit_utxo called!\n"); abort(); }
-/* Generated stub for towire */
-void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED)
-{ fprintf(stderr, "towire called!\n"); abort(); }
-/* Generated stub for towire_secp256k1_ecdsa_signature */
-void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED,
-			      const secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); }
-/* Generated stub for towire_sha256 */
-void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED)
-{ fprintf(stderr, "towire_sha256 called!\n"); abort(); }
-/* Generated stub for towire_u32 */
-void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED)
-{ fprintf(stderr, "towire_u32 called!\n"); abort(); }
-/* Generated stub for towire_u8 */
-void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED)
-{ fprintf(stderr, "towire_u8 called!\n"); abort(); }
-/* Generated stub for towire_u8_array */
-void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED)
-{ fprintf(stderr, "towire_u8_array called!\n"); abort(); }
-/* Generated stub for towire_wally_psbt */
-void towire_wally_psbt(u8 **pptr UNNEEDED, const struct wally_psbt *psbt UNNEEDED)
-{ fprintf(stderr, "towire_wally_psbt called!\n"); abort(); }
-/* AUTOGENERATED MOCKS END */
-
-int main(int argc, const char *argv[])
-{
-	struct privkey p;
-	struct pubkey pub;
-	struct bitcoin_signature sig;
-	struct sha256_double h;
-	u8 **wit;
-	size_t weight;
-
-	common_setup(argv[0]);
-	chainparams = chainparams_for_network("bitcoin");
-
-	memset(&h, 0, sizeof(h));
-	memset(&p, 1, sizeof(p));
-	sign_hash(&p, &h, &sig.s);
-	sig.sighash_type = SIGHASH_ALL;
-	pubkey_from_privkey(&p, &pub);
-
-	wit = bitcoin_witness_2of2(tmpctx, &sig, &sig, &pub, &pub);
-
-	/* 1 byte for num witnesses, one per witness element */
-	weight = 1;
-
-	/* Two signatures, slightly overestimated to be 73 bytes each,
-	 * while the actual witness will often be smaller.*/
-        /* BOLT #03:
-         * Signatures are 73 bytes long (the maximum length).
-         */
-	weight += 2 + 2;
-
-	for (size_t i = 0; i < tal_count(wit); i++)
-		weight += 1 + tal_bytelen(wit[i]);
-	assert(bitcoin_tx_2of2_input_witness_weight() == weight);
-
-	common_shutdown();
-	return 0;
-}

bitcoin/test/run-tx-encode.c

@@ -1,25 +1,22 @@
-#include "config.h"
 #include <assert.h>
 #include <bitcoin/psbt.c>
 #include <bitcoin/shadouble.c>
-#include <bitcoin/signature.c>
 #include <bitcoin/tx.c>
 #include <bitcoin/varint.c>
 #include <ccan/str/hex/hex.h>
 #include <common/setup.h>
 #include <common/utils.h>
-#include <stdio.h>
 
 /* AUTOGENERATED MOCKS START */
+/* Generated stub for amount_asset_extract_value */
+u8 *amount_asset_extract_value(const tal_t *ctx UNNEEDED, struct amount_asset *asset UNNEEDED)
+{ fprintf(stderr, "amount_asset_extract_value called!\n"); abort(); }
 /* Generated stub for amount_asset_is_main */
 bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_is_main called!\n"); abort(); }
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
-/* Generated stub for amount_feerate */
- bool amount_feerate(u32 *feerate UNNEEDED, struct amount_sat fee UNNEEDED, size_t weight UNNEEDED)
-{ fprintf(stderr, "amount_feerate called!\n"); abort(); }
 /* Generated stub for amount_sat */
 struct amount_sat amount_sat(u64 satoshis UNNEEDED)
 { fprintf(stderr, "amount_sat called!\n"); abort(); }
@@ -48,53 +45,61 @@ struct amount_sat amount_tx_fee(u32 fee_per_kw UNNEEDED, size_t weight UNNEEDED)
 /* Generated stub for fromwire */
 const u8 *fromwire(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, void *copy UNNEEDED, size_t n UNNEEDED)
 { fprintf(stderr, "fromwire called!\n"); abort(); }
+/* Generated stub for fromwire_amount_sat */
+struct amount_sat fromwire_amount_sat(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_amount_sat called!\n"); abort(); }
 /* Generated stub for fromwire_fail */
 void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_fail called!\n"); abort(); }
-/* Generated stub for fromwire_secp256k1_ecdsa_signature */
-void fromwire_secp256k1_ecdsa_signature(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-					secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "fromwire_secp256k1_ecdsa_signature called!\n"); abort(); }
 /* Generated stub for fromwire_sha256 */
 void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED)
 { fprintf(stderr, "fromwire_sha256 called!\n"); abort(); }
+/* Generated stub for fromwire_tal_arrn */
+u8 *fromwire_tal_arrn(const tal_t *ctx UNNEEDED,
+		       const u8 **cursor UNNEEDED, size_t *max UNNEEDED, size_t num UNNEEDED)
+{ fprintf(stderr, "fromwire_tal_arrn called!\n"); abort(); }
+/* Generated stub for fromwire_u16 */
+u16 fromwire_u16(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_u16 called!\n"); abort(); }
 /* Generated stub for fromwire_u32 */
 u32 fromwire_u32(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_u32 called!\n"); abort(); }
-/* Generated stub for fromwire_u8 */
-u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); }
-/* Generated stub for fromwire_u8_array */
-void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED)
-{ fprintf(stderr, "fromwire_u8_array called!\n"); abort(); }
 /* Generated stub for is_anchor_witness_script */
 bool is_anchor_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
 { fprintf(stderr, "is_anchor_witness_script called!\n"); abort(); }
-/* Generated stub for is_to_remote_anchored_witness_script */
-bool is_to_remote_anchored_witness_script(const u8 *script UNNEEDED, size_t script_len UNNEEDED)
-{ fprintf(stderr, "is_to_remote_anchored_witness_script called!\n"); abort(); }
+/* Generated stub for is_p2sh */
+bool is_p2sh(const u8 *script UNNEEDED, struct ripemd160 *addr UNNEEDED)
+{ fprintf(stderr, "is_p2sh called!\n"); abort(); }
+/* Generated stub for is_p2wpkh */
+bool is_p2wpkh(const u8 *script UNNEEDED, struct bitcoin_address *addr UNNEEDED)
+{ fprintf(stderr, "is_p2wpkh called!\n"); abort(); }
+/* Generated stub for is_p2wsh */
+bool is_p2wsh(const u8 *script UNNEEDED, struct sha256 *addr UNNEEDED)
+{ fprintf(stderr, "is_p2wsh called!\n"); abort(); }
 /* Generated stub for pubkey_to_der */
 void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UNNEEDED)
 { fprintf(stderr, "pubkey_to_der called!\n"); abort(); }
 /* Generated stub for pubkey_to_hash160 */
 void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
 { fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
+/* Generated stub for script_push_bytes */
+void script_push_bytes(u8 **scriptp UNNEEDED, const void *mem UNNEEDED, size_t len UNNEEDED)
+{ fprintf(stderr, "script_push_bytes called!\n"); abort(); }
 /* Generated stub for scriptpubkey_p2wsh */
 u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
 { fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }
-/* Generated stub for towire_secp256k1_ecdsa_signature */
-void towire_secp256k1_ecdsa_signature(u8 **pptr UNNEEDED,
-			      const secp256k1_ecdsa_signature *signature UNNEEDED)
-{ fprintf(stderr, "towire_secp256k1_ecdsa_signature called!\n"); abort(); }
+/* Generated stub for towire_amount_sat */
+void towire_amount_sat(u8 **pptr UNNEEDED, const struct amount_sat sat UNNEEDED)
+{ fprintf(stderr, "towire_amount_sat called!\n"); abort(); }
 /* Generated stub for towire_sha256 */
 void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED)
 { fprintf(stderr, "towire_sha256 called!\n"); abort(); }
+/* Generated stub for towire_u16 */
+void towire_u16(u8 **pptr UNNEEDED, u16 v UNNEEDED)
+{ fprintf(stderr, "towire_u16 called!\n"); abort(); }
 /* Generated stub for towire_u32 */
 void towire_u32(u8 **pptr UNNEEDED, u32 v UNNEEDED)
 { fprintf(stderr, "towire_u32 called!\n"); abort(); }
-/* Generated stub for towire_u8 */
-void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED)
-{ fprintf(stderr, "towire_u8 called!\n"); abort(); }
 /* Generated stub for towire_u8_array */
 void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNEEDED)
 { fprintf(stderr, "towire_u8_array called!\n"); abort(); }

bitcoin/tx.c

@@ -1,15 +1,21 @@
-#include "config.h"
 #include <assert.h>
+#include <bitcoin/block.h>
+#include <bitcoin/chainparams.h>
 #include <bitcoin/psbt.h>
 #include <bitcoin/script.h>
 #include <bitcoin/tx.h>
 #include <ccan/cast/cast.h>
+#include <ccan/crypto/sha256/sha256.h>
+#include <ccan/endian/endian.h>
+#include <ccan/mem/mem.h>
+#include <ccan/read_write_all/read_write_all.h>
 #include <ccan/str/hex/hex.h>
-#include <ccan/tal/str/str.h>
-#include <common/utils.h>
-#include <wally_psbt.h>
+#include <common/type_to_string.h>
+#include <stdio.h>
 #include <wire/wire.h>
 
+#define SEGREGATED_WITNESS_FLAG 0x1
+
 struct bitcoin_tx_output *new_tx_output(const tal_t *ctx,
 					struct amount_sat amount,
 					const u8 *script)
@@ -20,15 +26,13 @@ struct bitcoin_tx_output *new_tx_output(const tal_t *ctx,
 	return output;
 }
 
-struct wally_tx_output *wally_tx_output(const tal_t *ctx,
-					const u8 *script,
+struct wally_tx_output *wally_tx_output(const u8 *script,
 					struct amount_sat amount)
 {
 	u64 satoshis = amount.satoshis; /* Raw: wally API */
 	struct wally_tx_output *output;
 	int ret;
 
-	tal_wally_start();
 	if (chainparams->is_elements) {
 		u8 value[9];
 		ret = wally_tx_confidential_value_from_satoshi(satoshis, value,
@@ -37,10 +41,8 @@ struct wally_tx_output *wally_tx_output(const tal_t *ctx,
 		ret = wally_tx_elements_output_init_alloc(
 		    script, tal_bytelen(script), chainparams->fee_asset_tag, 33,
 		    value, sizeof(value), NULL, 0, NULL, 0, NULL, 0, &output);
-		if (ret != WALLY_OK) {
-			output = NULL;
-			goto done;
-		}
+		if (ret != WALLY_OK)
+			return NULL;
 
 		/* Cheat a bit by also setting the numeric satoshi value,
 		 * otherwise we end up converting a number of times */
@@ -48,19 +50,14 @@ struct wally_tx_output *wally_tx_output(const tal_t *ctx,
 	} else {
 		ret = wally_tx_output_init_alloc(satoshis, script,
 						 tal_bytelen(script), &output);
-		if (ret != WALLY_OK) {
-			output = NULL;
-			goto done;
-		}
+		if (ret != WALLY_OK)
+			return NULL;
 	}
-
-done:
-	tal_wally_end_onto(ctx, output, struct wally_tx_output);
 	return output;
 }
 
 int bitcoin_tx_add_output(struct bitcoin_tx *tx, const u8 *script,
-			  const u8 *wscript, struct amount_sat amount)
+			  u8 *wscript, struct amount_sat amount)
 {
 	size_t i = tx->wtx->num_outputs;
 	struct wally_tx_output *output;
@@ -72,21 +69,17 @@ int bitcoin_tx_add_output(struct bitcoin_tx *tx, const u8 *script,
 	assert(tx->wtx != NULL);
 	assert(chainparams);
 
-	output = wally_tx_output(NULL, script, amount);
+	output = wally_tx_output(script, amount);
 	assert(output);
-	tal_wally_start();
 	ret = wally_tx_add_output(tx->wtx, output);
 	assert(ret == WALLY_OK);
-	tal_wally_end(tx->wtx);
 
 	psbt_out = psbt_add_output(tx->psbt, output, i);
 	if (wscript) {
-		tal_wally_start();
 		ret = wally_psbt_output_set_witness_script(psbt_out,
 							   wscript,
 							   tal_bytelen(wscript));
 		assert(ret == WALLY_OK);
-		tal_wally_end(tx->psbt);
 	}
 
 	wally_tx_output_free(output);
@@ -95,25 +88,21 @@ int bitcoin_tx_add_output(struct bitcoin_tx *tx, const u8 *script,
 	return i;
 }
 
-void bitcoin_tx_remove_output(struct bitcoin_tx *tx, size_t outnum)
+int bitcoin_tx_add_multi_outputs(struct bitcoin_tx *tx,
+				 struct bitcoin_tx_output **outputs)
 {
-	int ret;
-	ret = wally_tx_remove_output(tx->wtx, outnum);
-	assert(ret == WALLY_OK);
-	ret = wally_psbt_remove_output(tx->psbt, outnum);
-	assert(ret == WALLY_OK);
-}
+	for (size_t j = 0; j < tal_count(outputs); j++)
+		bitcoin_tx_add_output(tx, outputs[j]->script,
+				      NULL, outputs[j]->amount);
 
-bool elements_wtx_output_is_fee(const struct wally_tx *tx, int outnum)
-{
-	assert(outnum < tx->num_outputs);
-	return chainparams->is_elements &&
-		tx->outputs[outnum].script_len == 0;
+	return tx->wtx->num_outputs;
 }
 
 bool elements_tx_output_is_fee(const struct bitcoin_tx *tx, int outnum)
 {
-	return elements_wtx_output_is_fee(tx->wtx, outnum);
+	assert(outnum < tx->wtx->num_outputs);
+	return chainparams->is_elements &&
+	       tx->wtx->outputs[outnum].script_len == 0;
 }
 
 struct amount_sat bitcoin_tx_compute_fee_w_inputs(const struct bitcoin_tx *tx,
@@ -170,7 +159,7 @@ static int elements_tx_add_fee_output(struct bitcoin_tx *tx)
 	int pos;
 
 	/* If we aren't using elements, we don't add explicit fee outputs */
-	if (!chainparams->is_elements)
+	if (!chainparams->is_elements || amount_sat_eq(fee, AMOUNT_SAT(0)))
 		return -1;
 
 	/* Try to find any existing fee output */
@@ -190,79 +179,84 @@ static int elements_tx_add_fee_output(struct bitcoin_tx *tx)
 void bitcoin_tx_set_locktime(struct bitcoin_tx *tx, u32 locktime)
 {
 	tx->wtx->locktime = locktime;
-	tx->psbt->fallback_locktime = locktime;
-	tx->psbt->has_fallback_locktime = true;
+	tx->psbt->tx->locktime = locktime;
 }
 
-/* FIXME Stolen from psbt_append_input; export? */
-static struct wally_tx_input *wally_tx_input_from_outpoint_sequence(const struct bitcoin_outpoint *outpoint,
-			u32 sequence)
-{
-	struct wally_tx_input *tx_in;
-    if (chainparams->is_elements) {
-        if (wally_tx_elements_input_init_alloc(outpoint->txid.shad.sha.u.u8,
-                               sizeof(outpoint->txid.shad.sha.u.u8),
-                               outpoint->n,
-                               sequence, NULL, 0,
-                               NULL,
-                               NULL, 0,
-                               NULL, 0, NULL, 0,
-                               NULL, 0, NULL, 0,
-                               NULL, 0, NULL,
-                               &tx_in) != WALLY_OK)
-            abort();
-    } else {
-        if (wally_tx_input_init_alloc(outpoint->txid.shad.sha.u.u8,
-                          sizeof(outpoint->txid.shad.sha.u.u8),
-                          outpoint->n,
-                          sequence, NULL, 0, NULL,
-                          &tx_in) != WALLY_OK)
-            abort();
-    }
-	return tx_in;
-}
-
-int bitcoin_tx_add_input(struct bitcoin_tx *tx,
-			 const struct bitcoin_outpoint *outpoint,
-			 u32 sequence, const u8 *scriptSig,
+int bitcoin_tx_add_input(struct bitcoin_tx *tx, const struct bitcoin_txid *txid,
+			 u32 outnum, u32 sequence, const u8 *scriptSig,
 			 struct amount_sat amount, const u8 *scriptPubkey,
 			 const u8 *input_wscript)
 {
+	struct wally_tx_input *input;
 	int wally_err;
-	int input_num = tx->wtx->num_inputs;
-	struct wally_tx_input *tx_input;
+	size_t i;
 
-	psbt_append_input(tx->psbt, outpoint,
-			  sequence, scriptSig,
-			  input_wscript, NULL);
+	assert(tx->wtx != NULL);
+	i = tx->wtx->num_inputs;
+	wally_err = wally_tx_input_init_alloc(txid->shad.sha.u.u8,
+					      sizeof(struct bitcoin_txid),
+					      outnum, sequence,
+					      scriptSig, tal_bytelen(scriptSig),
+					      NULL /* Empty witness stack */,
+					      &input);
+	assert(wally_err == WALLY_OK);
+	input->features = chainparams->is_elements ? WALLY_TX_IS_ELEMENTS : 0;
+	wally_tx_add_input(tx->wtx, input);
+	psbt_add_input(tx->psbt, input, i);
 
 	if (input_wscript) {
-		scriptPubkey = scriptpubkey_p2wsh(tmpctx, input_wscript);
+		/* Add the prev output's data into the PSBT struct */
+		if (is_elements(chainparams)) {
+			struct amount_asset asset;
+			/*FIXME: persist asset tags */
+			asset = amount_sat_to_asset(
+					&amount,
+					chainparams->fee_asset_tag);
+			psbt_elements_input_init_witness(tx->psbt, i,
+							 input_wscript,
+							 &asset, NULL);
+		} else
+			psbt_input_set_prev_utxo_wscript(tx->psbt, i,
+							 input_wscript,
+							 amount);
+	} else if (scriptPubkey) {
+		if (is_p2wsh(scriptPubkey, NULL) ||
+			is_p2wpkh(scriptPubkey, NULL) ||
+			/* FIXME: assert that p2sh inputs are
+			 * witness/are accompanied by a
+			 * redeemscript+witnessscript */
+			is_p2sh(scriptPubkey, NULL)) {
+			/* the only way to get here currently with
+			 * a p2sh script is via a p2sh-p2wpkh script
+			 * that we've created ...*/
+			/* BIP0174:
+			 * ** Value: The entire transaction output in
+			 * network serialization which the
+			 * current input spends from.
+			 * This should only be present for
+			 * inputs which spend segwit outputs,
+			 * including P2SH embedded ones.
+			 */
+			if (is_elements(chainparams)) {
+				struct amount_asset asset;
+				/*FIXME: persist asset tags */
+				asset = amount_sat_to_asset(
+						&amount,
+						chainparams->fee_asset_tag);
+				/* FIXME: persist nonces */
+				psbt_elements_input_init(tx->psbt, i,
+							 scriptPubkey,
+							 &asset, NULL);
+			} else
+				psbt_input_set_prev_utxo(tx->psbt, i,
+							 scriptPubkey,
+							 amount);
+		}
 	}
 
-	assert(scriptPubkey);
-	psbt_input_set_wit_utxo(tx->psbt, input_num,
-				scriptPubkey, amount);
+	wally_tx_input_free(input);
 
-	tal_wally_start();
-	tx_input = wally_tx_input_from_outpoint_sequence(outpoint, sequence);
-	wally_err = wally_tx_add_input(tx->wtx,
-			       tx_input);
-	assert(wally_err == WALLY_OK);
-	wally_tx_input_free(tx_input);
-
-	/* scriptsig isn't actually stored in psbt input, so add that now */
-	wally_tx_set_input_script(tx->wtx, input_num,
-				  scriptSig, tal_bytelen(scriptSig));
-	tal_wally_end(tx->wtx);
-
-	if (is_elements(chainparams)) {
-		/* FIXME: persist asset tags */
-		amount_sat_to_asset(&amount,
-					    chainparams->fee_asset_tag);
-		/* FIXME: persist nonces */
-	}
-	return input_num;
+	return i;
 }
 
 bool bitcoin_tx_check(const struct bitcoin_tx *tx)
@@ -282,7 +276,6 @@ bool bitcoin_tx_check(const struct bitcoin_tx *tx)
 	if (written != tal_bytelen(newtx))
 		return false;
 
-	tal_free(newtx);
 	return true;
 }
 
@@ -298,24 +291,47 @@ void bitcoin_tx_output_set_amount(struct bitcoin_tx *tx, int outnum,
 		assert(ret == WALLY_OK);
 	} else {
 		output->satoshi = satoshis;
+
+		/* update the global tx for the psbt also */
+		output = &tx->psbt->tx->outputs[outnum];
+		output->satoshi = satoshis;
 	}
-	wally_psbt_output_set_amount(&tx->psbt->outputs[outnum], satoshis);
+}
+
+const u8 *wally_tx_output_get_script(const tal_t *ctx,
+				     const struct wally_tx_output *output)
+{
+	if (output->script == NULL) {
+		/* This can happen for coinbase transactions and pegin
+		 * transactions */
+		return NULL;
+	}
+
+	return tal_dup_arr(ctx, u8, output->script, output->script_len, 0);
+}
+
+const u8 *bitcoin_tx_output_get_script(const tal_t *ctx,
+				       const struct bitcoin_tx *tx, int outnum)
+{
+	const struct wally_tx_output *output;
+	assert(outnum < tx->wtx->num_outputs);
+	output = &tx->wtx->outputs[outnum];
+
+	return wally_tx_output_get_script(ctx, output);
 }
 
 u8 *bitcoin_tx_output_get_witscript(const tal_t *ctx, const struct bitcoin_tx *tx,
 				    int outnum)
 {
 	struct wally_psbt_output *out;
-	const struct wally_map_item *output_witness_script;
 
 	assert(outnum < tx->psbt->num_outputs);
 	out = &tx->psbt->outputs[outnum];
 
-	output_witness_script = wally_map_get_integer(&out->psbt_fields, /* PSBT_OUT_WITNESS_SCRIPT */ 0x01);
-	if (output_witness_script->value_len == 0)
+	if (out->witness_script_len == 0)
 		return NULL;
 
-	return tal_dup_arr(ctx, u8, output_witness_script->value, output_witness_script->value_len, 0);
+	return tal_dup_arr(ctx, u8, out->witness_script, out->witness_script_len, 0);
 }
 
 struct amount_asset bitcoin_tx_output_get_amount(const struct bitcoin_tx *tx,
@@ -326,7 +342,7 @@ struct amount_asset bitcoin_tx_output_get_amount(const struct bitcoin_tx *tx,
 	return wally_tx_output_get_amount(&tx->wtx->outputs[outnum]);
 }
 
-void bitcoin_tx_output_get_amount_sat(const struct bitcoin_tx *tx, int outnum,
+void bitcoin_tx_output_get_amount_sat(struct bitcoin_tx *tx, int outnum,
 				      struct amount_sat *amount)
 {
 	struct amount_asset asset_amt;
@@ -341,7 +357,6 @@ void bitcoin_tx_input_set_witness(struct bitcoin_tx *tx, int innum,
 	struct wally_tx_witness_stack *stack = NULL;
 	size_t stack_size = tal_count(witness);
 
-	tal_wally_start();
 	/* Free any lingering witness */
 	if (witness) {
 		wally_tx_witness_stack_init_alloc(stack_size, &stack);
@@ -349,38 +364,51 @@ void bitcoin_tx_input_set_witness(struct bitcoin_tx *tx, int innum,
 			wally_tx_witness_stack_add(stack, witness[i],
 						   tal_bytelen(witness[i]));
 	}
-	tal_wally_end(tmpctx);
-
-	tal_wally_start();
 	wally_tx_set_input_witness(tx->wtx, innum, stack);
-	tal_wally_end(tx->wtx);
 
 	/* Also add to the psbt */
-	tal_wally_start();
-	wally_psbt_input_set_final_witness(&tx->psbt->inputs[innum], stack);
-	tal_wally_end(tx->psbt);
+	if (stack)
+		wally_psbt_input_set_final_witness(&tx->psbt->inputs[innum], stack);
+	else {
+		/* FIXME: libwally-psbt doesn't allow 'unsetting' of witness via
+		 * the set method at the moment, so we do it manually*/
+		struct wally_psbt_input *in = &tx->psbt->inputs[innum];
+		if (in->final_witness)
+			wally_tx_witness_stack_free(in->final_witness);
+		in->final_witness = NULL;
+	}
 
+	if (stack)
+		wally_tx_witness_stack_free(stack);
 	if (taken(witness))
-		tal_free(witness);
+	    tal_free(witness);
 }
 
 void bitcoin_tx_input_set_script(struct bitcoin_tx *tx, int innum, u8 *script)
 {
 	struct wally_psbt_input *in;
-
-	tal_wally_start();
 	wally_tx_set_input_script(tx->wtx, innum, script, tal_bytelen(script));
-	tal_wally_end(tx->wtx);
 
 	/* Also add to the psbt */
 	assert(innum < tx->psbt->num_inputs);
 	in = &tx->psbt->inputs[innum];
-	tal_wally_start();
 	wally_psbt_input_set_final_scriptsig(in, script, tal_bytelen(script));
-	tal_wally_end(tx->psbt);
 }
 
-/* FIXME: remove */
+const u8 *bitcoin_tx_input_get_witness(const tal_t *ctx,
+				       const struct bitcoin_tx *tx, int innum,
+				       int witnum)
+{
+	const u8 *witness_item;
+	struct wally_tx_witness_item *item;
+	assert(innum < tx->wtx->num_inputs);
+	assert(witnum < tx->wtx->inputs[innum].witness->num_items);
+	item = &tx->wtx->inputs[innum].witness->items[witnum];
+	witness_item =
+	    tal_dup_arr(ctx, u8, item->witness, item->witness_len, 0);
+	return witness_item;
+}
+
 void bitcoin_tx_input_get_txid(const struct bitcoin_tx *tx, int innum,
 			       struct bitcoin_txid *out)
 {
@@ -388,37 +416,11 @@ void bitcoin_tx_input_get_txid(const struct bitcoin_tx *tx, int innum,
 	wally_tx_input_get_txid(&tx->wtx->inputs[innum], out);
 }
 
-void bitcoin_tx_input_get_outpoint(const struct bitcoin_tx *tx,
-				   int innum,
-				   struct bitcoin_outpoint *outpoint)
-{
-	assert(innum < tx->wtx->num_inputs);
-	wally_tx_input_get_outpoint(&tx->wtx->inputs[innum], outpoint);
-}
-
-void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
-				   const struct bitcoin_outpoint *outpoint)
-{
-	struct wally_tx_input *in;
-	assert(innum < tx->wtx->num_inputs);
-
-	in = &tx->wtx->inputs[innum];
-	CROSS_TYPE_ASSIGNMENT(&in->txhash, &outpoint->txid);
-	in->index = outpoint->n;
-}
-
-/* FIXME: remove */
 void wally_tx_input_get_txid(const struct wally_tx_input *in,
 			     struct bitcoin_txid *txid)
 {
-	CROSS_TYPE_ASSIGNMENT(txid, &in->txhash);
-}
-
-void wally_tx_input_get_outpoint(const struct wally_tx_input *in,
-				 struct bitcoin_outpoint *outpoint)
-{
-	wally_tx_input_get_txid(in, &outpoint->txid);
-	outpoint->n = in->index;
+	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+	memcpy(txid, in->txhash, sizeof(struct bitcoin_txid));
 }
 
 /* BIP144:
@@ -458,31 +460,12 @@ u8 *linearize_wtx(const tal_t *ctx, const struct wally_tx *wtx)
 	return arr;
 }
 
-size_t wally_tx_weight(const struct wally_tx *wtx)
-{
-	size_t weight;
-	int ret = wally_tx_get_weight(wtx, &weight);
-	assert(ret == WALLY_OK);
-	return weight;
-}
-
 size_t bitcoin_tx_weight(const struct bitcoin_tx *tx)
 {
-	size_t extra;
-	size_t num_witnesses;
-
-	/* If we don't have witnesses *yet*, libwally doesn't encode
-	 * in BIP 141 style, omitting the flag and marker bytes */
-	wally_tx_get_witness_count(tx->wtx, &num_witnesses);
-	if (num_witnesses == 0) {
-		if (chainparams->is_elements)
-			extra = 7;
-		else
-			extra = 2;
-	} else
-		extra = 0;
-
-	return extra + wally_tx_weight(tx->wtx);
+	size_t weight;
+	int ret = wally_tx_get_weight(tx->wtx, &weight);
+	assert(ret == WALLY_OK);
+	return weight;
 }
 
 void wally_txid(const struct wally_tx *wtx, struct bitcoin_txid *txid)
@@ -530,12 +513,12 @@ struct bitcoin_tx *bitcoin_tx(const tal_t *ctx,
 	if (chainparams->is_elements)
 		output_count += 1;
 
-	tal_wally_start();
-	wally_tx_init_alloc(WALLY_TX_VERSION_2, nlocktime, input_count, output_count,
+	wally_tx_init_alloc(WALLY_TX_VERSION_2, 0, input_count, output_count,
 			    &tx->wtx);
 	tal_add_destructor(tx, bitcoin_tx_destroy);
-	tal_wally_end_onto(tx, tx->wtx, struct wally_tx);
 
+	tx->wtx->locktime = nlocktime;
+	tx->wtx->version = 2;
 	tx->chainparams = chainparams;
 	tx->psbt = new_psbt(tx, tx->wtx);
 
@@ -548,31 +531,16 @@ void bitcoin_tx_finalize(struct bitcoin_tx *tx)
 	assert(bitcoin_tx_check(tx));
 }
 
-struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt TAKES)
+struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt STEALS)
 {
-	size_t locktime;
-
-	if (!taken(psbt))
-		psbt = clone_psbt(tmpctx, psbt);
-
-	wally_psbt_get_locktime(psbt, &locktime);
 	struct bitcoin_tx *tx = bitcoin_tx(ctx, chainparams,
-					   psbt->num_inputs,
-					   psbt->num_outputs,
-					   locktime);
+					   psbt->tx->num_inputs,
+					   psbt->tx->num_outputs,
+					   psbt->tx->locktime);
 	wally_tx_free(tx->wtx);
-
-	psbt_finalize(psbt);
-	tx->wtx = psbt_final_tx(tx, psbt);
-	if (!tx->wtx) {
-		tal_wally_start();
-		if (wally_psbt_extract(psbt, WALLY_PSBT_EXTRACT_NON_FINAL, &tx->wtx) != WALLY_OK) {
-			tx->wtx = NULL;
-		}
-		tal_wally_end_onto(tx, tx->wtx, struct wally_tx);
-		if (!tx->wtx)
-			return tal_free(tx);
-	}
+	tx->wtx = psbt_finalize(psbt, false);
+	if (!tx->wtx && wally_tx_clone_alloc(psbt->tx, 0, &tx->wtx) != WALLY_OK)
+		return NULL;
 
 	tal_free(tx->psbt);
 	tx->psbt = tal_steal(tx, psbt);
@@ -580,81 +548,31 @@ struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psb
 	return tx;
 }
 
-struct bitcoin_tx *clone_bitcoin_tx(const tal_t *ctx,
-				    const struct bitcoin_tx *tx)
-{
-	struct bitcoin_tx *newtx;
-
-	if (taken(tx))
-		return cast_const(struct bitcoin_tx *, tal_steal(ctx, tx));
-
-	newtx = tal(ctx, struct bitcoin_tx);
-
-	newtx->chainparams = tx->chainparams;
-
-	tal_wally_start();
-	if (wally_tx_clone_alloc(tx->wtx, 0, &newtx->wtx) != WALLY_OK)
-		newtx->wtx = NULL;
-	tal_wally_end_onto(newtx, newtx->wtx, struct wally_tx);
-	if (!newtx->wtx)
-		return tal_free(newtx);
-
-	newtx->psbt = clone_psbt(newtx, tx->psbt);
-	tal_add_destructor(newtx, bitcoin_tx_destroy);
-	return newtx;
-}
-
-static struct wally_tx *pull_wtx(const tal_t *ctx,
-				 const u8 **cursor,
-				 size_t *max)
+struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx, const u8 **cursor,
+				   size_t *max)
 {
+	size_t wsize;
 	int flags = WALLY_TX_FLAG_USE_WITNESS;
-	struct wally_tx *wtx;
+	struct bitcoin_tx *tx = tal(ctx, struct bitcoin_tx);
 
 	if (chainparams->is_elements)
 		flags |= WALLY_TX_FLAG_USE_ELEMENTS;
 
-	tal_wally_start();
-	if (wally_tx_from_bytes(*cursor, *max, flags, &wtx) != WALLY_OK) {
+	if (wally_tx_from_bytes(*cursor, *max, flags, &tx->wtx) != WALLY_OK) {
 		fromwire_fail(cursor, max);
-		wtx = tal_free(wtx);
-	}
-	tal_wally_end_onto(ctx, wtx, struct wally_tx);
-
-	if (wtx) {
-		size_t wsize;
-		wally_tx_get_length(wtx, flags, &wsize);
-		*cursor += wsize;
-		*max -= wsize;
-	}
-
-	return wtx;
-}
-
-struct bitcoin_tx *pull_bitcoin_tx_only(const tal_t *ctx, const u8 **cursor,
-					size_t *max)
-{
-	struct bitcoin_tx *tx = tal(ctx, struct bitcoin_tx);
-	tx->wtx = pull_wtx(tx, cursor, max);
-	if (!tx->wtx)
 		return tal_free(tx);
+	}
 
 	tal_add_destructor(tx, bitcoin_tx_destroy);
+
+	wally_tx_get_length(tx->wtx, flags, &wsize);
+
 	tx->chainparams = chainparams;
-	tx->psbt = NULL;
 
-	return tx;
-}
+	tx->psbt = new_psbt(tx, tx->wtx);
 
-struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx, const u8 **cursor,
-				   size_t *max)
-{
-	struct bitcoin_tx *tx = pull_bitcoin_tx_only(ctx, cursor, max);
-	if (tx) {
-		tx->psbt = new_psbt(tx, tx->wtx);
-		if (!tx->psbt)
-			return tal_free(tx);
-        }
+	*cursor += wsize;
+	*max -= wsize;
 	return tx;
 }
 
@@ -694,6 +612,18 @@ fail:
 	return NULL;
 }
 
+/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
+static void reverse_bytes(u8 *arr, size_t len)
+{
+	unsigned int i;
+
+	for (i = 0; i < len / 2; i++) {
+		unsigned char tmp = arr[i];
+		arr[i] = arr[len - 1 - i];
+		arr[len - 1 - i] = tmp;
+	}
+}
+
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid)
 {
@@ -711,7 +641,7 @@ bool bitcoin_txid_to_hex(const struct bitcoin_txid *txid,
 	return hex_encode(&rev, sizeof(rev), hexstr, hexstr_len);
 }
 
-char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx)
+static char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx)
 {
 	u8 *lin = linearize_tx(ctx, tx);
 	char *s = tal_hex(ctx, lin);
@@ -719,7 +649,7 @@ char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx)
 	return s;
 }
 
-char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
+static char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
 {
 	char *hexstr = tal_arr(ctx, char, hex_str_size(sizeof(*txid)));
 
@@ -727,15 +657,7 @@ char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
 	return hexstr;
 }
 
-char *fmt_bitcoin_outpoint(const tal_t *ctx,
-			   const struct bitcoin_outpoint *outpoint)
-{
-	return tal_fmt(ctx, "%s:%u",
-		       fmt_bitcoin_txid(tmpctx, &outpoint->txid),
-		       outpoint->n);
-}
-
-char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
+static char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
 {
 	u8 *lin = linearize_wtx(ctx, tx);
 	char *s = tal_hex(ctx, lin);
@@ -743,15 +665,9 @@ char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
 	return s;
 }
 
-char *fmt_sha256(const tal_t *ctx, const struct sha256 *sha256)
-{
-	return tal_hexstr(ctx, sha256, sizeof(*sha256));
-}
-
-char *fmt_ripemd160(const tal_t *ctx, const struct ripemd160 *ripemd160)
-{
-	return tal_hexstr(ctx, ripemd160, sizeof(*ripemd160));
-}
+REGISTER_TYPE_TO_STRING(bitcoin_tx, fmt_bitcoin_tx);
+REGISTER_TYPE_TO_STRING(bitcoin_txid, fmt_bitcoin_txid);
+REGISTER_TYPE_TO_STRING(wally_tx, fmt_wally_tx);
 
 void fromwire_bitcoin_txid(const u8 **cursor, size_t *max,
 			   struct bitcoin_txid *txid)
@@ -764,18 +680,13 @@ struct bitcoin_tx *fromwire_bitcoin_tx(const tal_t *ctx,
 {
 	struct bitcoin_tx *tx;
 
-	u32 len = fromwire_u32(cursor, max);
-	size_t start = *max;
-	tx = pull_bitcoin_tx_only(ctx, cursor, max);
+	tx = pull_bitcoin_tx(ctx, cursor, max);
 	if (!tx)
 		return fromwire_fail(cursor, max);
-	// Check that we consumed len bytes
-	if (start - *max != len)
-		return fromwire_fail(cursor, max);
 
+	/* pull_bitcoin_tx sets the psbt */
+	tal_free(tx->psbt);
 	tx->psbt = fromwire_wally_psbt(tx, cursor, max);
-	if (!tx->psbt)
-		return fromwire_fail(cursor, max);
 
 	return tx;
 }
@@ -785,37 +696,44 @@ void towire_bitcoin_txid(u8 **pptr, const struct bitcoin_txid *txid)
 	towire_sha256_double(pptr, &txid->shad);
 }
 
-void towire_bitcoin_outpoint(u8 **pptr, const struct bitcoin_outpoint *outp)
-{
-	towire_bitcoin_txid(pptr, &outp->txid);
-	towire_u32(pptr, outp->n);
-}
-
-void fromwire_bitcoin_outpoint(const u8 **cursor, size_t *max,
-			       struct bitcoin_outpoint *outp)
-{
-	fromwire_bitcoin_txid(cursor, max, &outp->txid);
-	outp->n = fromwire_u32(cursor, max);
-}
-
 void towire_bitcoin_tx(u8 **pptr, const struct bitcoin_tx *tx)
 {
 	u8 *lin = linearize_tx(tmpctx, tx);
-	towire_u32(pptr, tal_count(lin));
 	towire_u8_array(pptr, lin, tal_count(lin));
 
 	towire_wally_psbt(pptr, tx->psbt);
 }
 
+struct bitcoin_tx_output *fromwire_bitcoin_tx_output(const tal_t *ctx,
+						     const u8 **cursor, size_t *max)
+{
+	struct bitcoin_tx_output *output = tal(ctx, struct bitcoin_tx_output);
+	output->amount = fromwire_amount_sat(cursor, max);
+	u16 script_len = fromwire_u16(cursor, max);
+	output->script = fromwire_tal_arrn(output, cursor, max, script_len);
+	if (!*cursor)
+		return tal_free(output);
+	return output;
+}
+
+void towire_bitcoin_tx_output(u8 **pptr, const struct bitcoin_tx_output *output)
+{
+	towire_amount_sat(pptr, output->amount);
+	towire_u16(pptr, tal_count(output->script));
+	towire_u8_array(pptr, output->script, tal_count(output->script));
+}
+
 bool wally_tx_input_spends(const struct wally_tx_input *input,
-			   const struct bitcoin_outpoint *outpoint)
+			   const struct bitcoin_txid *txid,
+			   int outnum)
 {
 	/* Useful, as tx_part can have some NULL inputs */
 	if (!input)
 		return false;
-	if (!CROSS_TYPE_EQ(&outpoint->txid, &input->txhash))
+	BUILD_ASSERT(sizeof(*txid) == sizeof(input->txhash));
+	if (memcmp(txid, input->txhash, sizeof(*txid)) != 0)
 		return false;
-	return input->index == outpoint->n;
+	return input->index == outnum;
 }
 
 /* FIXME(cdecker) Make the caller pass in a reference to amount_asset, and
@@ -888,20 +806,19 @@ size_t bitcoin_tx_output_weight(size_t outscript_len)
 	return weight;
 }
 
-/* We grind signatures to get them down to 71 bytes */
+/* We grind signatures to get them down to 71 bytes (+1 for sighash flags) */
 size_t bitcoin_tx_input_sig_weight(void)
 {
-	return 1 + 71;
+	return 1 + 71 + 1;
 }
 
-/* Input weight */
-size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight)
+/* We only do segwit inputs, and we assume witness is sig + key  */
+size_t bitcoin_tx_simple_input_weight(bool p2sh)
 {
-	/* We assume < 253 witness elements */
-	size_t weight = 1 + witness_weight;
+	size_t weight;
 
 	/* Input weight: txid + index + sequence */
-	weight += (32 + 4 + 4) * 4;
+	weight = (32 + 4 + 4) * 4;
 
 	/* We always encode the length of the script, even if empty */
 	weight += 1 * 4;
@@ -910,6 +827,9 @@ size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight)
 	if (p2sh)
 		weight += 23 * 4;
 
+	/* Account for witness (1 byte count + sig + key) */
+	weight += 1 + (bitcoin_tx_input_sig_weight() + 1 + 33);
+
 	/* Elements inputs have 6 bytes of blank proofs attached. */
 	if (chainparams->is_elements)
 		weight += 6;
@@ -917,74 +837,14 @@ size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight)
 	return weight;
 }
 
-size_t bitcoin_tx_input_witness_weight(enum utxotype utxotype)
+struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw)
 {
-	switch (utxotype) {
-	case UTXO_P2SH_P2WPKH:
-	case UTXO_P2WPKH:
-		/* Account for witness (sig + key) */
-		return bitcoin_tx_input_sig_weight() + 1 + 33;
-	case UTXO_P2WSH_FROM_CLOSE:
-		/* BOLT #3:
-		 * #### `to_remote` Output
-		 *
-		 * If `option_anchors` applies to the commitment
-		 * transaction, the `to_remote` output is encumbered by a one
-		 * block csv lock.
-		 *    <remotepubkey> OP_CHECKSIGVERIFY 1 OP_CHECKSEQUENCEVERIFY
-		 *
-		 * The output is spent by an input with `nSequence` field set
-		 * to `1` and witness: <remote_sig>
-		 * Otherwise, this output is a simple P2WPKH to `remotepubkey`.
-		 */
-		/* In practice, these predate anchors, so: */
-		return 1 + 1 + bitcoin_tx_input_sig_weight();
-	case UTXO_P2TR:
-		return 1 + 64;
-	}
-	abort();
-}
-
-size_t bitcoin_tx_2of2_input_witness_weight(void)
-{
-        /* BOLT #03:
-         * Signatures are 73 bytes long (the maximum length).
-         */
-	return 1 + /* Prefix: 4 elements to push on stack */
-		(1 + 0) + /* [0]: witness-marker-and-flag */
-		(1 + 73) + /* [1] Party A signature and length prefix */
-		(1 + 73) + /* [2] Party B signature and length prefix */
-		(1 + 1 + /* [3] length prefix and numpushes (2) */
-		 1 + 33 + /* pubkey A (with prefix) */
-		 1 + 33 + /* pubkey B (with prefix) */
-		 1 + 1 /* num sigs required and checkmultisig */
-		);
-}
-
-size_t change_weight(void)
-{
-	return bitcoin_tx_output_weight(chainparams->is_elements ? BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN : BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
-}
-
-struct amount_sat change_fee(u32 feerate_perkw,	size_t total_weight)
-{
-	struct amount_sat fee;
+	size_t outweight;
 
 	/* Must be able to pay for its own additional weight */
-	/* Rounding can cause off by one errors, so we do this */
-	if (!amount_sat_sub(&fee,
-			    amount_tx_fee(feerate_perkw, change_weight() + total_weight),
-			    amount_tx_fee(feerate_perkw, total_weight)))
-		abort();
-	return fee;
-}
-
-struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw,
-				size_t total_weight)
-{
-	struct amount_sat fee = change_fee(feerate_perkw, total_weight);
-
-	if (!amount_sat_sub(&excess, excess, fee))
+	outweight = bitcoin_tx_output_weight(BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN);
+	if (!amount_sat_sub(&excess,
+			    excess, amount_tx_fee(feerate_perkw, outweight)))
 		return AMOUNT_SAT(0);
 
 	/* Must be non-dust */
@@ -993,18 +853,3 @@ struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw,
 
 	return excess;
 }
-
-u32 tx_feerate(const struct bitcoin_tx *tx)
-{
-
-	u32 feerate;
-
-	/* Fee should not overflow! */
-	if (!amount_feerate(&feerate,
-			    bitcoin_tx_compute_fee(tx),
-			    bitcoin_tx_weight(tx))) {
-		abort();
-	}
-
-	return feerate;
-}

bitcoin/tx.h

@@ -1,12 +1,14 @@
 #ifndef LIGHTNING_BITCOIN_TX_H
 #define LIGHTNING_BITCOIN_TX_H
 #include "config.h"
-#include <bitcoin/chainparams.h>
-#include <bitcoin/shadouble.h>
-#include <bitcoin/signature.h>
-#include <bitcoin/varint.h>
+#include "shadouble.h"
+#include "signature.h"
+#include "varint.h"
+#include <ccan/short_types/short_types.h>
 #include <ccan/structeq/structeq.h>
+#include <ccan/tal/tal.h>
 #include <common/amount.h>
+#include <wally_psbt.h>
 #include <wally_transaction.h>
 
 #define BITCOIN_TX_DEFAULT_SEQUENCE 0xFFFFFFFF
@@ -14,25 +16,14 @@
 /* BIP 125: Any nsequence < 0xFFFFFFFE is replacable.
  * And bitcoind uses this value. */
 #define BITCOIN_TX_RBF_SEQUENCE 0xFFFFFFFD
-
 struct wally_psbt;
-struct ripemd160;
 
 struct bitcoin_txid {
 	struct sha256_double shad;
 };
-
-struct bitcoin_outpoint {
-	struct bitcoin_txid txid;
-	u32 n;
-};
-
 /* Define bitcoin_txid_eq */
 STRUCTEQ_DEF(bitcoin_txid, 0, shad.sha.u);
 
-/* Define bitcoin_outpoint_eq */
-STRUCTEQ_DEF(bitcoin_outpoint, 0, txid.shad.sha.u, n);
-
 struct bitcoin_tx {
 	struct wally_tx *wtx;
 
@@ -48,17 +39,6 @@ struct bitcoin_tx_output {
 	u8 *script;
 };
 
-enum utxotype {
-	/* Obsolete: we used to have P2SH-wrapped outputs (removed in 24.02, though can still have old UTXOs) */
-	UTXO_P2SH_P2WPKH = 1,
-	/* "bech32" addresses */
-	UTXO_P2WPKH = 2,
-	/* Used for closing addresses: implies ->close_info is non-NULL */
-	UTXO_P2WSH_FROM_CLOSE = 3,
-	/* "p2tr" addresses. */
-	UTXO_P2TR = 4,
-};
-
 struct bitcoin_tx_output *new_tx_output(const tal_t *ctx,
 					struct amount_sat amount,
 					const u8 *script);
@@ -71,9 +51,8 @@ void wally_txid(const struct wally_tx *wtx, struct bitcoin_txid *txid);
 u8 *linearize_tx(const tal_t *ctx, const struct bitcoin_tx *tx);
 u8 *linearize_wtx(const tal_t *ctx, const struct wally_tx *wtx);
 
-/* Get weight of tx in Sipa; assumes it will have witnesses! */
+/* Get weight of tx in Sipa. */
 size_t bitcoin_tx_weight(const struct bitcoin_tx *tx);
-size_t wally_tx_weight(const struct wally_tx *wtx);
 
 /* Allocate a tx: you just need to fill in inputs and outputs (they're
  * zeroed with inputs' sequence_number set to FFFFFFFF) */
@@ -82,26 +61,10 @@ struct bitcoin_tx *bitcoin_tx(const tal_t *ctx,
 			      varint_t input_count, varint_t output_count,
 			      u32 nlocktime);
 
-/* Make a (deep) copy */
-struct bitcoin_tx *clone_bitcoin_tx(const tal_t *ctx,
-				    const struct bitcoin_tx *tx TAKES);
-
 /* This takes a raw bitcoin tx in hex. */
 struct bitcoin_tx *bitcoin_tx_from_hex(const tal_t *ctx, const char *hex,
 				       size_t hexlen);
 
-/* <sigh>.  Bitcoind represents hashes as little-endian for RPC. */
-static inline void reverse_bytes(u8 *arr, size_t len)
-{
-	unsigned int i;
-
-	for (i = 0; i < len / 2; i++) {
-		unsigned char tmp = arr[i];
-		arr[i] = arr[len - 1 - i];
-		arr[len - 1 - i] = tmp;
-	}
-}
-
 /* Parse hex string to get txid (reversed, a-la bitcoind). */
 bool bitcoin_txid_from_hex(const char *hexstr, size_t hexstr_len,
 			   struct bitcoin_txid *txid);
@@ -111,32 +74,26 @@ bool bitcoin_txid_to_hex(const struct bitcoin_txid *txid,
 			 char *hexstr, size_t hexstr_len);
 
 /* Create a bitcoin_tx from a psbt */
-struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx,
-					struct wally_psbt *psbt TAKES);
+struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt);
 
 /* Internal de-linearization functions. */
-/* Pull a bitcoin tx, and create a PSBT wrapper for it */
 struct bitcoin_tx *pull_bitcoin_tx(const tal_t *ctx,
 				   const u8 **cursor, size_t *max);
 
-/* Pull a bitcoin tx without creating a PSBT wrapper for it */
-struct bitcoin_tx *pull_bitcoin_tx_only(const tal_t *ctx,
-					const u8 **cursor, size_t *max);
-
 /* Helper to create a wally_tx_output: make sure to wally_tx_output_free!
  * Returns NULL if amount is extreme (wally doesn't like).
  */
-struct wally_tx_output *wally_tx_output(const tal_t *ctx,
-					const u8 *script,
+struct wally_tx_output *wally_tx_output(const u8 *script,
 					struct amount_sat amount);
 
 /* Add one output to tx. */
 int bitcoin_tx_add_output(struct bitcoin_tx *tx, const u8 *script,
-			  const u8 *wscript,
+			  u8 *wscript,
 			  struct amount_sat amount);
 
-/* Remove one output. */
-void bitcoin_tx_remove_output(struct bitcoin_tx *tx, size_t outnum);
+/* Add mutiple output to tx. */
+int bitcoin_tx_add_multi_outputs(struct bitcoin_tx *tx,
+				 struct bitcoin_tx_output **outputs);
 
 /* Set the locktime for a transaction */
 void bitcoin_tx_set_locktime(struct bitcoin_tx *tx, u32 locktime);
@@ -147,15 +104,15 @@ void bitcoin_tx_set_locktime(struct bitcoin_tx *tx, u32 locktime);
  * Passing in just the {input_wscript}, we'll generate the scriptPubkey for you.
  * In some cases we may not have the wscript, in which case the scriptPubkey
  * should be provided. We'll check that it's P2WSH before saving it */
-int bitcoin_tx_add_input(struct bitcoin_tx *tx,
-			 const struct bitcoin_outpoint *outpoint,
-			 u32 sequence, const u8 *scriptSig,
+int bitcoin_tx_add_input(struct bitcoin_tx *tx, const struct bitcoin_txid *txid,
+			 u32 outnum, u32 sequence, const u8 *scriptSig,
 			 struct amount_sat amount, const u8 *scriptPubkey,
 			 const u8 *input_wscript);
 
-/* This is useful because wally uses a raw byte array for txids */
+/* This helps is useful because wally uses a raw byte array for txids */
 bool wally_tx_input_spends(const struct wally_tx_input *input,
-			   const struct bitcoin_outpoint *outpoint);
+			   const struct bitcoin_txid *txid,
+			   int outnum);
 
 struct amount_asset
 wally_tx_output_get_amount(const struct wally_tx_output *output);
@@ -170,6 +127,25 @@ wally_tx_output_get_amount(const struct wally_tx_output *output);
 void bitcoin_tx_output_set_amount(struct bitcoin_tx *tx, int outnum,
 				  struct amount_sat amount);
 
+/**
+ * Helper to get the script of a script's output as a tal_arr
+ *
+ * Internally we use a `wally_tx` to represent the transaction. The script
+ * attached to a `wally_tx_output` is not a `tal_arr`, so in order to keep the
+ * comfort of being able to call `tal_bytelen` and similar on a script we just
+ * return a `tal_arr` clone of the original script.
+ */
+const u8 *bitcoin_tx_output_get_script(const tal_t *ctx, const struct bitcoin_tx *tx, int outnum);
+
+/**
+ * Helper to get the script of a script's output as a tal_arr
+ *
+ * The script attached to a `wally_tx_output` is not a `tal_arr`, so in order to keep the
+ * comfort of being able to call `tal_bytelen` and similar on a script we just
+ * return a `tal_arr` clone of the original script.
+ */
+const u8 *wally_tx_output_get_script(const tal_t *ctx,
+				     const struct wally_tx_output *output);
 /**
  * Helper to get a witness script for an output.
  */
@@ -180,7 +156,7 @@ u8 *bitcoin_tx_output_get_witscript(const tal_t *ctx, const struct bitcoin_tx *t
  * Internally we use a `wally_tx` to represent the transaction. The
  * satoshi amount isn't a struct amount_sat, so we need a conversion
  */
-void bitcoin_tx_output_get_amount_sat(const struct bitcoin_tx *tx, int outnum,
+void bitcoin_tx_output_get_amount_sat(struct bitcoin_tx *tx, int outnum,
 				      struct amount_sat *amount);
 /**
  * Helper to just get an amount_sat for the output amount.
@@ -202,27 +178,21 @@ void bitcoin_tx_input_set_witness(struct bitcoin_tx *tx, int innum,
  */
 void bitcoin_tx_input_set_script(struct bitcoin_tx *tx, int innum, u8 *script);
 
+/**
+ * Helper to get a witness as a tal_arr array.
+ */
+const u8 *bitcoin_tx_input_get_witness(const tal_t *ctx,
+				       const struct bitcoin_tx *tx, int innum,
+				       int witnum);
+
 /**
  * Wrap the raw txhash in the wally_tx_input into a bitcoin_txid
  */
-void bitcoin_tx_input_get_outpoint(const struct bitcoin_tx *tx,
-				   int innum,
-				   struct bitcoin_outpoint *outpoint);
-
 void bitcoin_tx_input_get_txid(const struct bitcoin_tx *tx, int innum,
 			       struct bitcoin_txid *out);
 void wally_tx_input_get_txid(const struct wally_tx_input *in,
 			     struct bitcoin_txid *txid);
 
-void wally_tx_input_get_outpoint(const struct wally_tx_input *in,
-				 struct bitcoin_outpoint *outpoint);
-
-/**
- * Overwrite the txhash and index in the wally_tx_input
- */
-void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
-				   const struct bitcoin_outpoint *outpoint);
-
 /**
  * Check a transaction for consistency.
  *
@@ -240,57 +210,16 @@ bool bitcoin_tx_check(const struct bitcoin_tx *tx);
  */
 void bitcoin_tx_finalize(struct bitcoin_tx *tx);
 
-/**
- * Returns true if the given outnum is a fee output
- */
-bool elements_wtx_output_is_fee(const struct wally_tx *tx, int outnum);
-
 /**
  * Returns true if the given outnum is a fee output
  */
 bool elements_tx_output_is_fee(const struct bitcoin_tx *tx, int outnum);
 
-/** Attempt to compute the elements overhead given a base bitcoin size.
- *
- * The overhead consists of 2 empty proofs for the transaction, 6 bytes of
- * proofs per input and 35 bytes per output. In addition the explicit fee
- * output will add 9 bytes and the per output overhead as well.
- */
-static inline size_t elements_tx_overhead(const struct chainparams *chainparams,
-					  size_t incount, size_t outcount)
-{
-	size_t overhead;
-
-	if (!chainparams->is_elements)
-		return 0;
-
-	/* Each transaction has surjection and rangeproof (both empty
-	 * for us as long as we use unblinded L-BTC transactions). */
-	overhead = 2 * 4;
-	/* For elements we also need to add the fee output and the
-	 * overhead for rangeproofs into the mix. */
-	overhead += (8 + 1) * 4; /* Bitcoin style output */
-
-	/* All outputs have a bit of elements overhead (incl fee) */
-	overhead += (32 + 1 + 1 + 1) * 4 * (outcount + 1); /* Elements added fields */
-
-	/* Inputs have 6 bytes of blank proofs attached. */
-	overhead += 6 * incount;
-
-	return overhead;
-}
-
 /**
  * Calculate the fees for this transaction
  */
 struct amount_sat bitcoin_tx_compute_fee(const struct bitcoin_tx *tx);
 
-/**
- * Calculate the feerate for this transaction (in perkw)
-*/
-u32 tx_feerate(const struct bitcoin_tx *tx);
-
-
 /*
  * Calculate the fees for this transaction, given a pre-computed input balance.
  *
@@ -305,20 +234,11 @@ void fromwire_bitcoin_txid(const u8 **cursor, size_t *max,
 			   struct bitcoin_txid *txid);
 struct bitcoin_tx *fromwire_bitcoin_tx(const tal_t *ctx,
 				       const u8 **cursor, size_t *max);
+struct bitcoin_tx_output *fromwire_bitcoin_tx_output(const tal_t *ctx,
+						     const u8 **cursor, size_t *max);
 void towire_bitcoin_txid(u8 **pptr, const struct bitcoin_txid *txid);
 void towire_bitcoin_tx(u8 **pptr, const struct bitcoin_tx *tx);
-void towire_bitcoin_outpoint(u8 **pptr, const struct bitcoin_outpoint *outp);
-void fromwire_bitcoin_outpoint(const u8 **cursor, size_t *max,
-			       struct bitcoin_outpoint *outp);
-char *fmt_bitcoin_tx(const tal_t *ctx, const struct bitcoin_tx *tx);
-char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid);
-char *fmt_bitcoin_outpoint(const tal_t *ctx,
-			   const struct bitcoin_outpoint *outpoint);
-char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx);
-
-/* For want of somewhere better to define them! */
-char *fmt_sha256(const tal_t *ctx, const struct sha256 *sha256);
-char *fmt_ripemd160(const tal_t *ctx, const struct ripemd160 *ripemd160);
+void towire_bitcoin_tx_output(u8 **pptr, const struct bitcoin_tx_output *output);
 
 /* Various weights of transaction parts. */
 size_t bitcoin_tx_core_weight(size_t num_inputs, size_t num_outputs);
@@ -327,45 +247,18 @@ size_t bitcoin_tx_output_weight(size_t outscript_len);
 /* Weight to push sig on stack. */
 size_t bitcoin_tx_input_sig_weight(void);
 
-/* Segwit input, but with parameter for witness weight (size).
- * witness_weight must include the varint_size() for each witness element,
- * but not the varint_size() for the number of elements. */
-size_t bitcoin_tx_input_weight(bool p2sh, size_t witness_weight);
-
-/* The witness weight */
-size_t bitcoin_tx_input_witness_weight(enum utxotype utxotype);
-
-/* The witness for our 2of2 input (closing or commitment tx). */
-size_t bitcoin_tx_2of2_input_witness_weight(void);
+/* We only do segwit inputs, and we assume witness is sig + key  */
+size_t bitcoin_tx_simple_input_weight(bool p2sh);
 
 /**
- * change_weight - what's the weight of a change output?
- */
-size_t change_weight(void);
-
-/**
- * change_fee - what's the cost to add a change output to this tx?
- * @feerate_perkw: feerate.
- * @total_weight: current weight of tx.
- *
- * We pass in the total_weight of the tx (up until this point) so as
- * to avoid any off-by-one errors with rounding the change fee (down)
- */
-struct amount_sat change_fee(u32 feerate_perkw,	size_t total_weight);
-
-/**
- * change_amount - Is it worth making a change output at this feerate?
+ * change_amount - Is it worth making a P2WPKH change output at this feerate?
  * @excess: input amount we have above the tx fee and other outputs.
  * @feerate_perkw: feerate.
- * @total_weight: current weight of tx.
- *
- * Change script is P2TR for Bitcoin, P2WPKH for Elements
  *
  * If it's not worth (or possible) to make change, returns AMOUNT_SAT(0).
  * Otherwise returns the amount of the change output to add (@excess minus
- * the change_fee()).
+ * the additional fee for the change output itself).
  */
-struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw,
-				size_t total_weight);
+struct amount_sat change_amount(struct amount_sat excess, u32 feerate_perkw);
 
 #endif /* LIGHTNING_BITCOIN_TX_H */

bitcoin/tx_parts.c

@@ -1,4 +1,3 @@
-#include "config.h"
 #include <assert.h>
 #include <bitcoin/tx_parts.h>
 #include <common/utils.h>
@@ -10,11 +9,71 @@ static void destroy_wally_tx_input(struct wally_tx_input *in)
 	wally_tx_input_free(in);
 }
 
+static struct wally_tx_input *clone_input(const tal_t *ctx,
+					  const struct wally_tx_input *src)
+{
+	struct wally_tx_input *in;
+	int ret;
+
+	if (is_elements(chainparams)) {
+		ret = wally_tx_elements_input_init_alloc
+			(src->txhash, sizeof(src->txhash),
+			 src->index, src->sequence,
+			 src->script, src->script_len,
+			 src->witness,
+			 src->blinding_nonce, sizeof(src->blinding_nonce),
+			 src->entropy, sizeof(src->entropy),
+			 src->issuance_amount, src->issuance_amount_len,
+			 src->inflation_keys, src->inflation_keys_len,
+			 src->issuance_amount_rangeproof,
+			 src->issuance_amount_rangeproof_len,
+			 src->inflation_keys_rangeproof,
+			 src->inflation_keys_rangeproof_len,
+			 src->pegin_witness,
+			 &in);
+	} else {
+		ret = wally_tx_input_init_alloc(src->txhash, sizeof(src->txhash),
+						src->index, src->sequence,
+						src->script, src->script_len,
+						src->witness, &in);
+	}
+	assert(ret == WALLY_OK);
+
+	tal_add_destructor(in, destroy_wally_tx_input);
+	return tal_steal(ctx, in);
+}
+
 static void destroy_wally_tx_output(struct wally_tx_output *out)
 {
 	wally_tx_output_free(out);
 }
 
+static struct wally_tx_output *clone_output(const tal_t *ctx,
+					  const struct wally_tx_output *src)
+{
+	struct wally_tx_output *out;
+	int ret;
+
+	if (is_elements(chainparams)) {
+		ret = wally_tx_elements_output_init_alloc
+			(src->script, src->script_len,
+			 src->asset, src->asset_len,
+			 src->value, src->value_len,
+			 src->nonce, src->nonce_len,
+			 src->surjectionproof, src->surjectionproof_len,
+			 src->rangeproof, src->rangeproof_len,
+			 &out);
+	} else {
+		ret = wally_tx_output_init_alloc(src->satoshi,
+						 src->script, src->script_len,
+						 &out);
+	}
+	assert(ret == WALLY_OK);
+
+	tal_add_destructor(out, destroy_wally_tx_output);
+	return tal_steal(ctx, out);
+}
+
 struct tx_parts *tx_parts_from_wally_tx(const tal_t *ctx,
 					const struct wally_tx *wtx,
 					int input, int output)
@@ -25,47 +84,21 @@ struct tx_parts *tx_parts_from_wally_tx(const tal_t *ctx,
 	txp->inputs = tal_arrz(txp, struct wally_tx_input *, wtx->num_inputs);
 	txp->outputs = tal_arrz(txp, struct wally_tx_output *, wtx->num_outputs);
 
-	tal_wally_start();
 	for (size_t i = 0; i < wtx->num_inputs; i++) {
 		if (input != -1 && input != i)
 			continue;
-		if (wally_tx_input_clone_alloc(&wtx->inputs[i],
-					       &txp->inputs[i]) != WALLY_OK)
-			abort();
-		tal_add_destructor(txp->inputs[i], destroy_wally_tx_input);
+		txp->inputs[i] = clone_input(txp->inputs, &wtx->inputs[i]);
 	}
 
 	for (size_t i = 0; i < wtx->num_outputs; i++) {
 		if (output != -1 && output != i)
 			continue;
-		if (wally_tx_output_clone_alloc(&wtx->outputs[i],
-						&txp->outputs[i]) != WALLY_OK)
-			abort();
-		tal_add_destructor(txp->outputs[i], destroy_wally_tx_output);
-
-		/* Cheat a bit by also setting the numeric satoshi
-		 * value, otherwise we end up converting a
-		 * number of times */
-		if (chainparams->is_elements) {
-			struct amount_asset asset;
-			struct amount_sat sats;
-			asset = wally_tx_output_get_amount(txp->outputs[i]);
-			/* FIXME: non l-btc assets */
-			assert(amount_asset_is_main(&asset));
-			sats = amount_asset_to_sat(&asset);
-			txp->outputs[i]->satoshi = sats.satoshis; /* Raw: wally conversion */
-		}
+		txp->outputs[i] = clone_output(txp->outputs, &wtx->outputs[i]);
 	}
-	tal_wally_end(txp);
 
 	return txp;
 }
 
-static void destroy_wally_tx_witness_stack(struct wally_tx_witness_stack *ws)
-{
-	wally_tx_witness_stack_free(ws);
-}
-
 /* FIXME: If libwally exposed their linearization code, we could use it */
 static struct wally_tx_witness_stack *
 fromwire_wally_tx_witness_stack(const tal_t *ctx,
@@ -80,7 +113,6 @@ fromwire_wally_tx_witness_stack(const tal_t *ctx,
 	if (num == 0)
 		return NULL;
 
-	tal_wally_start();
 	ret = wally_tx_witness_stack_init_alloc(num, &ws);
 	if (ret != WALLY_OK) {
 		fromwire_fail(cursor, max);
@@ -95,14 +127,10 @@ fromwire_wally_tx_witness_stack(const tal_t *ctx,
 		if (ret != WALLY_OK) {
 			wally_tx_witness_stack_free(ws);
 			fromwire_fail(cursor, max);
-			ws = NULL;
-			goto out;
+			return NULL;
 		}
 	}
 
-	tal_add_destructor(ws, destroy_wally_tx_witness_stack);
-out:
-	tal_wally_end_onto(ctx, ws, struct wally_tx_witness_stack);
 	return ws;
 }
 
@@ -144,7 +172,6 @@ static struct wally_tx_input *fromwire_wally_tx_input(const tal_t *ctx,
 		script = tal_free(script);
 	ws = fromwire_wally_tx_witness_stack(tmpctx, cursor, max);
 
-	tal_wally_start();
 	if (is_elements(chainparams)) {
 		u8 *blinding_nonce, *entropy, *issuance_amount,
 			*inflation_keys, *issuance_amount_rangeproof,
@@ -195,13 +222,11 @@ static struct wally_tx_input *fromwire_wally_tx_input(const tal_t *ctx,
 	}
 	if (ret != WALLY_OK) {
 		fromwire_fail(cursor, max);
-		in = NULL;
-	} else {
-		tal_add_destructor(in, destroy_wally_tx_input);
+		return NULL;
 	}
 
-	tal_wally_end_onto(ctx, in, struct wally_tx_input);
-	return in;
+	tal_add_destructor(in, destroy_wally_tx_input);
+	return tal_steal(ctx, in);
 }
 
 static struct wally_tx_output *fromwire_wally_tx_output(const tal_t *ctx,
@@ -215,7 +240,6 @@ static struct wally_tx_output *fromwire_wally_tx_output(const tal_t *ctx,
 	script = fromwire_tal_arrn(tmpctx,
 				   cursor, max, fromwire_u32(cursor, max));
 
-	tal_wally_start();
 	if (is_elements(chainparams)) {
 		u8 *asset, *value, *nonce, *surjectionproof, *rangeproof;
 
@@ -242,9 +266,6 @@ static struct wally_tx_output *fromwire_wally_tx_output(const tal_t *ctx,
 			 surjectionproof, tal_bytelen(surjectionproof),
 			 rangeproof, tal_bytelen(rangeproof),
 			 &out);
-
-		/* As a convenience, we sent the value over as satoshis */
-		out->satoshi = fromwire_u64(cursor, max);
 	} else {
 		u64 satoshi;
 		satoshi = fromwire_u64(cursor, max);
@@ -254,13 +275,11 @@ static struct wally_tx_output *fromwire_wally_tx_output(const tal_t *ctx,
 	}
 	if (ret != WALLY_OK) {
 		fromwire_fail(cursor, max);
-		out = NULL;
-	} else {
-		tal_add_destructor(out, destroy_wally_tx_output);
+		return NULL;
 	}
-	tal_wally_end_onto(ctx, out, struct wally_tx_output);
 
-	return out;
+	tal_add_destructor(out, destroy_wally_tx_output);
+	return tal_steal(ctx, out);
 }
 
 static void towire_wally_tx_input(u8 **pptr, const struct wally_tx_input *in)
@@ -312,8 +331,6 @@ static void towire_wally_tx_output(u8 **pptr, const struct wally_tx_output *out)
 				out->surjectionproof_len);
 		towire_u32(pptr, out->rangeproof_len);
 		towire_u8_array(pptr, out->rangeproof, out->rangeproof_len);
-		/* Copy the value over, as a convenience */
-		towire_u64(pptr, out->satoshi);
 	} else {
 		towire_u64(pptr, out->satoshi);
 	}

bitcoin/tx_parts.h

@@ -5,6 +5,7 @@
 #define LIGHTNING_BITCOIN_TX_PARTS_H
 #include "config.h"
 #include <bitcoin/tx.h>
+#include <wally_transaction.h>
 
 struct tx_parts {
 	/* The txid of this transacation */

bitcoin/varint.c

@@ -1,5 +1,4 @@
-#include "config.h"
-#include <bitcoin/varint.h>
+#include "varint.h"
 
 size_t varint_size(varint_t v)
 {

ccan/README

@@ -1,3 +1,3 @@
 CCAN imported from http://ccodearchive.net.
 
-CCAN version: init-2593-gca094039
+CCAN version: init-2500-gcbc7cbf1

ccan/ccan/autodata/_info

@@ -0,0 +1,111 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * autodata - stash pointers in your binary for automatic registration
+ *
+ * This code allows declarations in your source which you can gather
+ * together at runtime to form tables.  This is often used in place of
+ * having a central registration function or table.
+ *
+ * Note that this technique does not work in general for shared libaries,
+ * only for code compiled into a binary.
+ *
+ * License: BSD-MIT
+ *
+ * Example:
+ * // Distributed commandline option registration (note: ccan/opt is better!)
+ * #include <ccan/autodata/autodata.h>
+ * #include <stdio.h>
+ * #include <unistd.h>
+ * #include <stdbool.h>
+ * #include <err.h>
+ *
+ * static bool verbose = false;
+ *
+ * // This would normally be in a header, so any C file can use it.
+ * struct option {
+ *	char c;
+ *	bool takes_arg;
+ *	bool (*cb)(char *optarg);
+ * };
+ * AUTODATA_TYPE(options, struct option);
+ * #define REGISTER_OPTION(optstruct) \
+ *		AUTODATA(options, (optstruct))
+ *
+ * // Now a few examples (could be anywhere in source)
+ * static bool verbose_cb(char *unused)
+ * {
+ *	verbose = true;
+ *      return true;
+ * }
+ * static struct option dash_v = { 'v', false, verbose_cb };
+ * REGISTER_OPTION(&dash_v);
+ *
+ * static bool chdir_cb(char *dir)
+ * {
+ *	if (verbose)
+ *		printf("chdir to %s. ", dir);
+ *	if (chdir(dir) != 0)
+ *		return false;
+ *      return true;
+ * }
+ * static struct option dash_C = { 'C', true, chdir_cb };
+ * REGISTER_OPTION(&dash_C);
+ *
+ * int main(int argc, char *argv[])
+ * {
+ * 	struct option **opts;
+ *	size_t i, num;
+ *	int o;
+ *	char *optstring, *p;
+ *
+ *	// Gather together all the registered options.
+ *	opts = autodata_get(options, &num);
+ *
+ *	// Make pretty string for getopt().
+ *	p = optstring = malloc(num * 2 + 1);
+ *	for (i = 0; i < num; i++) {
+ *		*(p++) = opts[i]->c;
+ *		if (opts[i]->takes_arg)
+ *			*(p++) = ':';
+ *	}
+ *	*p = '\0';
+ *
+ *	while ((o = getopt(argc, argv, optstring)) != -1) {
+ *		if (o == '?')
+ *			exit(1);
+ *		// Call callback in matching option.
+ *		for (i = 0; i < num; i++) {
+ *			if (opts[i]->c == o) {
+ *				if (!opts[i]->cb(optarg))
+ *					err(1, "parsing -%c", o);
+ *				break;
+ *			}
+ *		}
+ *	}
+ *	// free up gathered option table.
+ *	autodata_free(opts);
+ *
+ *	if (verbose)
+ *		printf("verbose mode on\n");
+ *	return 0;
+ * }
+ * // Given "-v" outputs "verbose mode on\n"
+ * // Given "-v -C /" outputs "chdir to /. verbose mode on\n"
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/compiler\n");
+		printf("ccan/ptr_valid\n");
+		return 0;
+	}
+
+	return 1;
+}

ccan/ccan/autodata/autodata.c

@@ -0,0 +1,80 @@
+// Licensed under BSD-MIT: See LICENSE.
+#include "autodata.h"
+#include <stdint.h>
+#include <unistd.h>
+#include <string.h>
+
+#if HAVE_SECTION_START_STOP
+void *autodata_get_section(void *start, void *stop, size_t *nump)
+{
+	*nump = (void **)(stop) - (void **)(start);
+	return start;
+}
+
+void autodata_free(void *table UNNEEDED)
+{
+}
+#else
+#include <ccan/ptr_valid/ptr_valid.h>
+
+void *autodata_make_table(const void *example, const char *name, size_t *nump)
+{
+	const char *start, *end, *tag;
+	struct ptr_valid_batch batch;
+	const void *const magic = (void *)AUTODATA_MAGIC;
+	void **table = NULL;
+	char first_magic;
+
+	if (!ptr_valid_batch_start(&batch))
+		return NULL;
+
+	/* Get range to search. */
+	for (start = (char *)((intptr_t)example & ~(getpagesize() - 1));
+	     ptr_valid_batch(&batch, start-getpagesize(), 1, sizeof(void *),
+			     false);
+	     start -= getpagesize());
+
+	for (end = (char *)((intptr_t)example & ~(getpagesize() - 1));
+	     ptr_valid_batch(&batch, end, 1, sizeof(void *), false);
+	     end += getpagesize());
+
+	*nump = 0;
+	first_magic = *(char *)&magic;
+	for (tag = memchr(start, first_magic, end - start);
+	     tag;
+	     tag = memchr(tag+1, first_magic, end - (tag + 1))) {
+		void *adata[4];
+
+		/* We can read 4 void *'s here? */
+		if (tag + sizeof(adata) > end)
+			continue;
+
+		memcpy(adata, tag, sizeof(adata));
+
+		/* False match? */
+		if (adata[0] != (void *)AUTODATA_MAGIC || adata[1] != tag)
+			continue;
+
+		/* OK, check name. */
+		if (!ptr_valid_batch_string(&batch, adata[3])
+		    || strcmp(name, adata[3]) != 0)
+			continue;
+
+		if (!ptr_valid_batch_read(&batch, (char *)adata[2]))
+			continue;
+
+		table = realloc(table, sizeof(void *) * (*nump + 1));
+		if (!table)
+			break;
+		table[*nump] = adata[2];
+		(*nump)++;
+	}
+	ptr_valid_batch_end(&batch);
+	return table;
+}
+
+void autodata_free(void *table)
+{
+	free(table);
+}
+#endif

ccan/ccan/autodata/autodata.h

@@ -0,0 +1,109 @@
+// Licensed under BSD-MIT: See LICENSE.
+#ifndef CCAN_AUTODATA_H
+#define CCAN_AUTODATA_H
+#include "config.h"
+#include <ccan/compiler/compiler.h>
+#include <stdlib.h>
+
+#if HAVE_SECTION_START_STOP
+
+/**
+ * AUTODATA_TYPE - declare the type for a given autodata name.
+ * @name: the name for this set of autodata
+ * @type: the type this autodata points to
+ *
+ * This macro is usually placed in a header: it must precede any
+ * autodata functions in the file.
+ *
+ * Example:
+ *	#include <ccan/autodata/autodata.h>
+ *
+ *	// My set of char pointers.
+ *	AUTODATA_TYPE(names, char);
+ */
+#define AUTODATA_TYPE(name, type)					\
+	typedef type autodata_##name##_;				\
+	extern type *__start_xautodata_##name[], *__stop_xautodata_##name[]
+
+/**
+ * AUTODATA - add a pointer to this autodata set
+ * @name: the name of the set of autodata
+ * @ptr: the compile-time-known pointer
+ *
+ * This embeds @ptr into the binary, with the tag corresponding to
+ * @name (which must look like a valid identifier, no punctuation!).
+ * The type of @ptr must match that given by AUTODATA_TYPE.  It is
+ * usually a file-level declaration.
+ *
+ * Example:
+ *	// Put two char pointers into the names AUTODATA set.
+ *	AUTODATA(names, "Arabella");
+ *	AUTODATA(names, "Alex");
+ */
+#define AUTODATA(name, ptr) \
+	static const autodata_##name##_ *NEEDED		\
+	__attribute__((section("xautodata_" #name)))	\
+	AUTODATA_VAR_(name, __LINE__) = (ptr);
+
+/**
+ * autodata_get - get an autodata set
+ * @name: the name of the set of autodata
+ * @nump: the number of items in the set.
+ *
+ * This extract the embedded pointers matching @name.  It may fail
+ * if malloc() fails, or if there is no AUTODATA at all.
+ *
+ * The return will be a pointer to an array of @type pointers (from
+ * AUTODATA_TYPE).
+ *
+ * Example:
+ *	static void print_embedded_names(void)
+ *	{
+ *		unsigned int i;
+ *		size_t num;
+ *		char **n = autodata_get(names, &num);
+ *
+ *		for (i = 0; i < num; i++)
+ *			printf("%s\n", n[i]);
+ *	}
+ */
+#define autodata_get(name, nump)					\
+	((autodata_##name##_ **)					\
+	 autodata_get_section(__start_xautodata_##name,			\
+			      __stop_xautodata_##name, (nump)))
+#endif /* HAVE_SECTION_START_STOP */
+
+/**
+ * autodata_free - free the table returned by autodata_get()
+ * @p: the table.
+ */
+void autodata_free(void *p);
+
+/* Internal functions. */
+#define AUTODATA_VAR__(name, line) autodata_##name##_##line
+#define AUTODATA_VAR_(name, line) AUTODATA_VAR__(name, line)
+
+#if HAVE_SECTION_START_STOP
+void *autodata_get_section(void *start, void *stop, size_t *nump);
+#else
+#define AUTODATA_TYPE(name, type)					\
+	typedef type autodata_##name##_;				\
+	static const void *autodata_##name##_ex = &autodata_##name##_ex
+
+#define AUTODATA_MAGIC ((long)0xFEEDA10DA7AF00D5ULL)
+#define AUTODATA(name, ptr)						\
+	static const autodata_##name##_ *NEEDED				\
+	AUTODATA_VAR_(name, __LINE__)[4] =				\
+	{ (void *)AUTODATA_MAGIC,					\
+	  (void *)&AUTODATA_VAR_(name, __LINE__),			\
+	  (ptr),							\
+	  (void *)#name }
+
+#define autodata_get(name, nump)					\
+	((autodata_##name##_ **)					\
+	 autodata_make_table(&autodata_##name##_ex, #name, (nump)))
+
+void *autodata_make_table(const void *example, const char *name, size_t *nump);
+#endif
+
+#endif /* CCAN_AUTODATA_H */

ccan/ccan/autodata/test/helper.c

@@ -0,0 +1,6 @@
+/* Check that linking together works. */
+#include <ccan/autodata/autodata.h>
+
+AUTODATA_TYPE(autostrings, char);
+
+AUTODATA(autostrings, "helper");

ccan/ccan/autodata/test/run-fools.c

@@ -0,0 +1,71 @@
+#include <ccan/autodata/autodata.h>
+/* Include the C files directly. */
+#include <ccan/autodata/autodata.c>
+#include <ccan/tap/tap.h>
+
+AUTODATA_TYPE(autostrings, char);
+
+AUTODATA(autostrings, "genuine");
+
+#if !HAVE_SECTION_START_STOP
+/* These are all fake, to test the various failure paths. */
+/* Hopefully fake_alpha or fake_omega will test run-past-end. */
+static const void *NEEDED fake_alpha[] = { (void *)AUTODATA_MAGIC };
+
+/* Wrong magic in the middle. */
+static const void *NEEDED fake1[] = { (void *)(AUTODATA_MAGIC ^ 0x10000),
+				      (void *)&fake1,
+				      "fake1",
+				      (void *)"autostrings" };
+
+/* Wrong self pointer. */
+static const void *NEEDED fake2[] = { (void *)AUTODATA_MAGIC,
+				      (void *)&fake1,
+				      "fake2",
+				      (void *)"autostrings" };
+
+/* Wrong name. */
+static const void *NEEDED fake3[] = { (void *)AUTODATA_MAGIC,
+				      (void *)&fake3,
+				      "fake3",
+				      (void *)"autostrings2" };
+
+/* Invalid self-pointer. */
+static const void *NEEDED fake4[] = { (void *)AUTODATA_MAGIC,
+				      (void *)1UL,
+				      "fake4",
+				      (void *)"autostrings" };
+
+/* Invalid name pointer */
+static const void *NEEDED fake5[] = { (void *)AUTODATA_MAGIC,
+				      (void *)&fake5,
+				      "fake5",
+				      (void *)1UL };
+
+/* Invalid contents pointer */
+static const void *NEEDED fake6[] = { (void *)AUTODATA_MAGIC,
+				      (void *)&fake6,
+				      (char *)1UL,
+				      (void *)"autostrings" };
+
+static const void *NEEDED fake_omega[] = { (void *)AUTODATA_MAGIC };
+#endif
+
+int main(void)
+{
+	char **table;
+	size_t num;
+
+	/* This is how many tests you plan to run */
+	plan_tests(2);
+
+	table = autodata_get(autostrings, &num);
+	ok1(num == 2);
+	ok1((!strcmp(table[0], "genuine") && !strcmp(table[1], "helper"))
+	    || (!strcmp(table[1], "genuine") && !strcmp(table[0], "helper")));
+
+	autodata_free(table);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/autodata/test/run.c

@@ -0,0 +1,41 @@
+#include <ccan/autodata/autodata.h>
+/* Include the C files directly. */
+#include <ccan/autodata/autodata.c>
+#include <ccan/tap/tap.h>
+
+AUTODATA_TYPE(autostrings, char);
+
+AUTODATA(autostrings, "hello");
+AUTODATA(autostrings, "world");
+
+int main(void)
+{
+	char **table;
+	size_t num;
+	int i, hello = -1, world = -1, helper = -1;
+
+	/* This is how many tests you plan to run */
+	plan_tests(4);
+
+	table = autodata_get(autostrings, &num);
+	ok1(num == 3);
+
+	for (i = 0; i < num; i++) {
+		if (strcmp(table[i], "hello") == 0)
+			hello = i;
+		else if (strcmp(table[i], "world") == 0)
+			world = i;
+		else if (strcmp(table[i], "helper") == 0)
+			helper = i;
+		else
+			fail("Unknown entry %s", table[i]);
+	}
+	ok1(hello != -1);
+	ok1(world != -1);
+	ok1(helper != -1);
+
+	autodata_free(table);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/base64/_info

@@ -1,41 +0,0 @@
-#include "config.h"
-
-/**
- * base64 - base64 encoding and decoding (rfc4648).
- *
- * base64 encoding is used to encode data in a 7-bit clean manner.
- *  Commonly used for escaping data before encapsulation or transfer
- *
- * Example:
- *      #include <stdio.h>
- *      #include <string.h>
- *      #include <ccan/base64/base64.h>
- *
- *      int main(int argc, char *argv[])
- *      {
- *      	char *base64_encoded_string;
- *      	int i;
- *
- *      	// print the base64-encoded form of the program arguments
- *      	for(i=1;i<argc;i++) {
- *			size_t unencoded_length = strlen(argv[i]);
- *      		size_t encoded_length = base64_encoded_length(unencoded_length);
- *      		base64_encoded_string = malloc(encoded_length);
- *      		base64_encode(base64_encoded_string, encoded_length,
- *				 argv[i], unencoded_length);
- *      		printf("%s\n", base64_encoded_string);
- *      		free(base64_encoded_string);
- *      	}
- *
- *      	return 0;
- *      }
- *
- * License: BSD-MIT
- */
-int main(int argc, char *argv[])
-{
-	if (argc != 2)
-		return 1;
-
-	return 1;
-}

ccan/ccan/base64/base64.c

@@ -1,253 +0,0 @@
-/* Licensed under BSD-MIT - see LICENSE file for details */
-#include "base64.h"
-
-#include <errno.h>
-#include <string.h>
-#include <assert.h>
-#include <stdint.h>
-
-/**
- * sixbit_to_b64 - maps a 6-bit value to the base64 alphabet
- * @param map A base 64 map (see base64_init_map)
- * @param sixbit Six-bit value to map
- * @return a base 64 character
- */
-static char sixbit_to_b64(const base64_maps_t *maps, const uint8_t sixbit)
-{
-	assert(sixbit <= 63);
-
-	return maps->encode_map[(unsigned char)sixbit];
-}
-
-/**
- * sixbit_from_b64 - maps a base64-alphabet character to its 6-bit value
- * @param maps A base 64 maps structure (see base64_init_maps)
- * @param sixbit Six-bit value to map
- * @return a six-bit value
- */
-static int8_t sixbit_from_b64(const base64_maps_t *maps,
-			      const unsigned char b64letter)
-{
-	int8_t ret;
-
-	ret = maps->decode_map[(unsigned char)b64letter];
-	if (ret == (int8_t)'\xff') {
-		errno = EDOM;
-		return -1;
-	}
-
-	return ret;
-}
-
-bool base64_char_in_alphabet(const base64_maps_t *maps, const char b64char)
-{
-	return (maps->decode_map[(const unsigned char)b64char] != (signed char)'\xff');
-}
-
-void base64_init_maps(base64_maps_t *dest, const char src[64])
-{
-	unsigned char i;
-
-	memcpy(dest->encode_map,src,64);
-	memset(dest->decode_map,0xff,256);
-	for (i=0; i<64; i++) {
-		dest->decode_map[(unsigned char)src[i]] = i;
-	}
-}
-
-size_t base64_encoded_length(size_t srclen)
-{
-	return ((srclen + 2) / 3) * 4;
-}
-
-void base64_encode_triplet_using_maps(const base64_maps_t *maps,
-				      char dest[4], const char src[3])
-{
-	char a = src[0];
-	char b = src[1];
-	char c = src[2];
-
-	dest[0] = sixbit_to_b64(maps, (a & 0xfc) >> 2);
-	dest[1] = sixbit_to_b64(maps, ((a & 0x3) << 4) | ((b & 0xf0) >> 4));
-	dest[2] = sixbit_to_b64(maps, ((c & 0xc0) >> 6) | ((b & 0xf) << 2));
-	dest[3] = sixbit_to_b64(maps, c & 0x3f);
-}
-
-void base64_encode_tail_using_maps(const base64_maps_t *maps, char dest[4],
-				   const char *src, const size_t srclen)
-{
-	char longsrc[3] = { 0 };
-
-	assert(srclen <= 3);
-
-	memcpy(longsrc, src, srclen);
-	base64_encode_triplet_using_maps(maps, dest, longsrc);
-	memset(dest+1+srclen, '=', 3-srclen);
-}
-
-ssize_t base64_encode_using_maps(const base64_maps_t *maps,
-				 char *dest, const size_t destlen,
-				 const char *src, const size_t srclen)
-{
-	size_t src_offset = 0;
-	size_t dest_offset = 0;
-
-	if (destlen < base64_encoded_length(srclen)) {
-		errno = EOVERFLOW;
-		return -1;
-	}
-
-	while (srclen - src_offset >= 3) {
-		base64_encode_triplet_using_maps(maps, &dest[dest_offset], &src[src_offset]);
-		src_offset += 3;
-		dest_offset += 4;
-	}
-
-	if (src_offset < srclen) {
-		base64_encode_tail_using_maps(maps, &dest[dest_offset], &src[src_offset], srclen-src_offset);
-		dest_offset += 4;
-	}
-
-	memset(&dest[dest_offset], '\0', destlen-dest_offset);
-
-	return dest_offset;
-}
-
-size_t base64_decoded_length(size_t srclen)
-{
-	return ((srclen+3)/4*3);
-}
-
-ssize_t base64_decode_quartet_using_maps(const base64_maps_t *maps, char dest[3],
-				     const char src[4])
-{
-	signed char a;
-	signed char b;
-	signed char c;
-	signed char d;
-
-	a = sixbit_from_b64(maps, src[0]);
-	b = sixbit_from_b64(maps, src[1]);
-	c = sixbit_from_b64(maps, src[2]);
-	d = sixbit_from_b64(maps, src[3]);
-
-	if ((a == -1) || (b == -1) || (c == -1) || (d == -1)) {
-		return -1;
-	}
-
-	dest[0] = (a << 2) | (b >> 4);
-	dest[1] = ((b & 0xf) << 4) | (c >> 2);
-	dest[2] = ((c & 0x3) << 6) | d;
-
-	return 0;
-}
-
-
-ssize_t base64_decode_tail_using_maps(const base64_maps_t *maps, char dest[3],
-				  const char * src, const size_t srclen)
-{
-	char longsrc[4];
-	int quartet_result;
-	size_t insize = srclen;
-
-	while (insize != 0 &&
-	       src[insize-1] == '=') { /* throw away padding symbols */
-		insize--;
-	}
-	if (insize == 0) {
-		return 0;
-	}
-	if (insize == 1) {
-		/* the input is malformed.... */
-		errno = EINVAL;
-		return -1;
-	}
-	memcpy(longsrc, src, insize);
-	memset(longsrc+insize, 'A', 4-insize);
-	quartet_result = base64_decode_quartet_using_maps(maps, dest, longsrc);
-	if (quartet_result == -1) {
-		return -1;
-	}
-
-	return insize - 1;
-}
-
-ssize_t base64_decode_using_maps(const base64_maps_t *maps,
-				 char *dest, const size_t destlen,
-				 const char *src, const size_t srclen)
-{
-	ssize_t dest_offset = 0;
-	ssize_t i;
-	ssize_t more;
-
-	if (destlen < base64_decoded_length(srclen)) {
-		errno = EOVERFLOW;
-		return -1;
-	}
-
-	for(i=0; srclen - i > 4; i+=4) {
-		if (base64_decode_quartet_using_maps(maps, &dest[dest_offset], &src[i]) == -1) {
-			return -1;
-		}
-		dest_offset += 3;
-	}
-
-	more = base64_decode_tail_using_maps(maps, &dest[dest_offset], &src[i], srclen - i);
-	if (more == -1) {
-		return -1;
-	}
-	dest_offset += more;
-
-	memset(&dest[dest_offset], '\0', destlen-dest_offset);
-
-	return dest_offset;
-}
-
-
-
-
-/**
- * base64_maps_rfc4648 - pregenerated maps struct for rfc4648
- */
-const base64_maps_t base64_maps_rfc4648 = {
-  "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",
-
-  "\xff\xff\xff\xff\xff" /* 0 */					\
-  "\xff\xff\xff\xff\xff" /* 5 */					\
-  "\xff\xff\xff\xff\xff" /* 10 */					\
-  "\xff\xff\xff\xff\xff" /* 15 */					\
-  "\xff\xff\xff\xff\xff" /* 20 */					\
-  "\xff\xff\xff\xff\xff" /* 25 */					\
-  "\xff\xff\xff\xff\xff" /* 30 */					\
-  "\xff\xff\xff\xff\xff" /* 35 */					\
-  "\xff\xff\xff\x3e\xff" /* 40 */					\
-  "\xff\xff\x3f\x34\x35" /* 45 */					\
-  "\x36\x37\x38\x39\x3a" /* 50 */					\
-  "\x3b\x3c\x3d\xff\xff" /* 55 */					\
-  "\xff\xff\xff\xff\xff" /* 60 */					\
-  "\x00\x01\x02\x03\x04" /* 65 A */					\
-  "\x05\x06\x07\x08\x09" /* 70 */					\
-  "\x0a\x0b\x0c\x0d\x0e" /* 75 */					\
-  "\x0f\x10\x11\x12\x13" /* 80 */					\
-  "\x14\x15\x16\x17\x18" /* 85 */					\
-  "\x19\xff\xff\xff\xff" /* 90 */					\
-  "\xff\xff\x1a\x1b\x1c" /* 95 */					\
-  "\x1d\x1e\x1f\x20\x21" /* 100 */					\
-  "\x22\x23\x24\x25\x26" /* 105 */					\
-  "\x27\x28\x29\x2a\x2b" /* 110 */					\
-  "\x2c\x2d\x2e\x2f\x30" /* 115 */					\
-  "\x31\x32\x33\xff\xff" /* 120 */					\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 125 */			\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 155 */			\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 185 */			\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 215 */			\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"				\
-  "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 245 */
-};

ccan/ccan/base64/base64.h

@@ -1,241 +0,0 @@
-/* Licensed under BSD-MIT - see LICENSE file for details */
-#ifndef CCAN_BASE64_H
-#define CCAN_BASE64_H
-
-#include <stddef.h>
-#include <stdbool.h>
-#include <sys/types.h>
-
-/**
- * base64_maps_t - structure to hold maps for encode/decode
- */
-typedef struct {
-	char encode_map[64];
-	signed char decode_map[256];
-} base64_maps_t;
-
-/**
- * base64_encoded_length - Calculate encode buffer length
- * @param srclen the size of the data to be encoded
- * @note add 1 to this to get null-termination
- * @return Buffer length required for encode
- */
-size_t base64_encoded_length(size_t srclen);
-
-/**
- * base64_decoded_length - Calculate decode buffer length
- * @param srclen Length of the data to be decoded
- * @note This does not return the size of the decoded data!  see base64_decode
- * @return Minimum buffer length for safe decode
- */
-size_t base64_decoded_length(size_t srclen);
-
-/**
- * base64_init_maps - populate a base64_maps_t based on a supplied alphabet
- * @param dest A base64 maps object
- * @param src Alphabet to populate the maps from (e.g. base64_alphabet_rfc4648)
- */
-void base64_init_maps(base64_maps_t *dest, const char src[64]);
-
-
-/**
- * base64_encode_triplet_using_maps - encode 3 bytes into base64 using a specific alphabet
- * @param maps Maps to use for encoding (see base64_init_maps)
- * @param dest Buffer containing 3 bytes
- * @param src Buffer containing 4 characters
- */
-void base64_encode_triplet_using_maps(const base64_maps_t *maps,
-				      char dest[4], const char src[3]);
-
-/**
- * base64_encode_tail_using_maps - encode the final bytes of a source using a specific alphabet
- * @param maps Maps to use for encoding (see base64_init_maps)
- * @param dest Buffer containing 4 bytes
- * @param src Buffer containing srclen bytes
- * @param srclen Number of bytes (<= 3) to encode in src
- */
-void base64_encode_tail_using_maps(const base64_maps_t *maps, char dest[4],
-				   const char *src, size_t srclen);
-
-/**
- * base64_encode_using_maps - encode a buffer into base64 using a specific alphabet
- * @param maps Maps to use for encoding (see base64_init_maps)
- * @param dest Buffer to encode into
- * @param destlen Length of dest
- * @param src Buffer to encode
- * @param srclen Length of the data to encode
- * @return Number of encoded bytes set in dest. -1 on error (and errno set)
- * @note dest will be nul-padded to destlen (past any required padding)
- * @note sets errno = EOVERFLOW if destlen is too small
- */
-ssize_t base64_encode_using_maps(const base64_maps_t *maps,
-				 char *dest, size_t destlen,
-				 const char *src, size_t srclen);
-
-/*
- * base64_char_in_alphabet - returns true if character can be part of an encoded string
- * @param maps A base64 maps object (see base64_init_maps)
- * @param b64char Character to check
- */
-bool base64_char_in_alphabet(const base64_maps_t *maps, char b64char);
-
-/**
- * base64_decode_using_maps - decode a base64-encoded string using a specific alphabet
- * @param maps A base64 maps object (see base64_init_maps)
- * @param dest Buffer to decode into
- * @param destlen length of dest
- * @param src the buffer to decode
- * @param srclen the length of the data to decode
- * @return Number of decoded bytes set in dest. -1 on error (and errno set)
- * @note dest will be nul-padded to destlen
- * @note sets errno = EOVERFLOW if destlen is too small
- * @note sets errno = EDOM if src contains invalid characters
- */
-ssize_t base64_decode_using_maps(const base64_maps_t *maps,
-				 char *dest, size_t destlen,
-				 const char *src, size_t srclen);
-
-/**
- * base64_decode_quartet_using_maps - decode 4 bytes from base64 using a specific alphabet
- * @param maps A base64 maps object (see base64_init_maps)
- * @param dest Buffer containing 3 bytes
- * @param src Buffer containing 4 bytes
- * @return Number of decoded bytes set in dest. -1 on error (and errno set)
- * @note sets errno = EDOM if src contains invalid characters
- */
-ssize_t base64_decode_quartet_using_maps(const base64_maps_t *maps,
-				         char dest[3], const char src[4]);
-
-/**
- * base64_decode_tail_using_maps - decode the final bytes of a base64 string using a specific alphabet
- * @param maps A base64 maps object (see base64_init_maps)
- * @param dest Buffer containing 3 bytes
- * @param src Buffer containing 4 bytes - padded with '=' as required
- * @param srclen Number of bytes to decode in src
- * @return Number of decoded bytes set in dest. -1 on error (and errno set)
- * @note sets errno = EDOM if src contains invalid characters
- * @note sets errno = EINVAL if src is an invalid base64 tail
- */
-ssize_t base64_decode_tail_using_maps(const base64_maps_t *maps, char dest[3],
-				      const char *src, size_t srclen);
-
-
-/* the rfc4648 functions: */
-
-extern const base64_maps_t base64_maps_rfc4648;
-
-/**
- * base64_encode - Encode a buffer into base64 according to rfc4648
- * @param dest Buffer to encode into
- * @param destlen Length of the destination buffer
- * @param src Buffer to encode
- * @param srclen Length of the data to encode
- * @return Number of encoded bytes set in dest. -1 on error (and errno set)
- * @note dest will be nul-padded to destlen (past any required padding)
- * @note sets errno = EOVERFLOW if destlen is too small
- *
- * This function encodes src according to http://tools.ietf.org/html/rfc4648
- *
- * Example:
- *	size_t encoded_length;
- *	char dest[100];
- *	const char *src = "This string gets encoded";
- *	encoded_length = base64_encode(dest, sizeof(dest), src, strlen(src));
- *	printf("Returned data of length %zd @%p\n", encoded_length, &dest);
- */
-static inline
-ssize_t base64_encode(char *dest, size_t destlen,
-		      const char *src, size_t srclen)
-{
-	return base64_encode_using_maps(&base64_maps_rfc4648,
-					dest, destlen, src, srclen);
-}
-
-/**
- * base64_encode_triplet - encode 3 bytes into base64 according to rfc4648
- * @param dest Buffer containing 4 bytes
- * @param src Buffer containing 3 bytes
- */
-static inline
-void base64_encode_triplet(char dest[4], const char src[3])
-{
-	base64_encode_triplet_using_maps(&base64_maps_rfc4648, dest, src);
-}
-
-/**
- * base64_encode_tail - encode the final bytes of a source according to rfc4648
- * @param dest Buffer containing 4 bytes
- * @param src Buffer containing srclen bytes
- * @param srclen Number of bytes (<= 3) to encode in src
- */
-static inline
-void base64_encode_tail(char dest[4], const char *src, size_t srclen)
-{
-	base64_encode_tail_using_maps(&base64_maps_rfc4648, dest, src, srclen);
-}
-
-
-/**
- * base64_decode - decode An rfc4648 base64-encoded string
- * @param dest Buffer to decode into
- * @param destlen Length of the destination buffer
- * @param src Buffer to decode
- * @param srclen Length of the data to decode
- * @return Number of decoded bytes set in dest. -1 on error (and errno set)
- * @note dest will be nul-padded to destlen
- * @note sets errno = EOVERFLOW if destlen is too small
- * @note sets errno = EDOM if src contains invalid characters
- *
- * This function decodes the buffer according to
- * http://tools.ietf.org/html/rfc4648
- *
- * Example:
- *	size_t decoded_length;
- *	char ret[100];
- *	const char *src = "Zm9vYmFyYmF6";
- *	decoded_length = base64_decode(ret, sizeof(ret), src, strlen(src));
- *	printf("Returned data of length %zd @%p\n", decoded_length, &ret);
- */
-static inline
-ssize_t base64_decode(char *dest, size_t destlen,
-		      const char *src, size_t srclen)
-{
-	return base64_decode_using_maps(&base64_maps_rfc4648,
-					dest, destlen, src, srclen);
-}
-
-/**
- * base64_decode_quartet - decode the first 4 characters in src into dest
- * @param dest Buffer containing 3 bytes
- * @param src Buffer containing 4 characters
- * @return Number of decoded bytes set in dest. -1 on error (and errno set)
- * @note sets errno = EDOM if src contains invalid characters
- */
-static inline
-ssize_t base64_decode_quartet(char dest[3], const char src[4])
-{
-	return base64_decode_quartet_using_maps(&base64_maps_rfc4648,
-						dest, src);
-}
-
-/**
- * @brief decode the final bytes of a base64 string from src into dest
- * @param dest Buffer containing 3 bytes
- * @param src Buffer containing 4 bytes - padded with '=' as required
- * @param srclen Number of bytes to decode in src
- * @return Number of decoded bytes set in dest. -1 on error (and errno set)
- * @note sets errno = EDOM if src contains invalid characters
- * @note sets errno = EINVAL if src is an invalid base64 tail
- */
-static inline
-ssize_t base64_decode_tail(char dest[3], const char *src, size_t srclen)
-{
-	return base64_decode_tail_using_maps(&base64_maps_rfc4648,
-					     dest, src, srclen);
-}
-
-/* end rfc4648 functions */
-
-
-
-#endif /* CCAN_BASE64_H */

ccan/ccan/base64/test/moretap.h

@@ -1,96 +0,0 @@
-#ifndef _BASE64_MORETAP_H
-#define _BASE64_MORETAP_H
-
-#include <ccan/str/str.h>
-
-/**
- * is_str - OK if strings are equal
- * @e1: expression for the variable string
- * @e2: expression for the expected string
- *
- * If the strings are equal, the test passes.
- *
- * Example:
- *     is_str(give_me_a_fred(),"fred");
- */
-static void _is_str(char *got,const char *expected, const char *got_string, const char *expected_string, const char *func, const char *file, int line) {
-	if (streq(expected,got)) {
-		_gen_result(1, func, file, line,"%s eq %s",
-			    got_string,expected_string);
-	} else {
-		_gen_result(0, func, file, line,"%s eq %s",
-			    got_string,expected_string);
-		diag("Expected: %s",expected);
-		diag("     Got: %s",got);
-	}
-}
-# define is_str(got,expected) _is_str(got,expected,#got,#expected,__func__, __FILE__, __LINE__)
-
-
-/**
- * is_int - OK if arguments are equal when cast to integers
- * @e1: expression for the number
- * @e2: expression for the expected number
- *
- * If the numbers are equal, the test passes.
- *
- * Example:
- *     is_int(give_me_17(),17);
- */
-# define is_int(e1,e2 ...)						\
-  (((int)e1)==((int)e2) ?						\
-   _gen_result(1, __func__, __FILE__, __LINE__,"%s == %s",#e1,#e2) :	\
-   (_gen_result(0, __func__, __FILE__, __LINE__,"%s == %s",#e1,#e2)) || (diag("Expected: %d",e2),diag("     Got: %d",e1),0)) /* diag is void; note commas. */
-
-
-
-/**
- * is_mem - OK if arguments are identical up to length @e3
- * @e1: expression for the buffer
- * @e2: expression for the expected buffer
- * @e2: length to compare in buffers
- *
- * If the buffers are equal up to @e2, the test passes.
- *
- * Example:
- *     is_mem(give_me_foo(),"foo",3);
- */
-static void _is_mem(const char *got, const char *expected, const size_t len,
-	      const char *got_string, const char *expected_string, const char *len_string,
-	      const char *func, const char *file, int line) {
-	size_t offset = 0;
-
-	for (offset=0; offset<len; offset++) {
-		if (got[offset] != expected[offset]) {
-			_gen_result(0, func, file, line,"%s eq %s",got_string,expected_string);
-			/* diag("Expected: %s",e2); */
-			/* diag("     Got: %s",e1); */
-			diag("Buffers differ at offset %zd (got=0x%02x expected=0x%02x)",
-			     offset,got[offset],expected[offset]);
-			return;
-		}
-	}
-
-	_gen_result(1, __func__, __FILE__, __LINE__,"%s eq %s",
-		    expected_string,got_string);
-}
-# define is_mem(got,expected,len) \
-	_is_mem(got,expected,len,#got,#expected,#len,__func__, __FILE__, __LINE__)
-
-/**
- * is_size_t - OK if arguments are equal when cast to size_t
- * @e1: expression for the number
- * @e2: expression for the expected number
- *
- * If the numbers are equal, the test passes.
- *
- * Example:
- *     is_size_t(give_me_17(),17);
- */
-# define is_size_t(e1,e2 ...)						\
-  ((size_t)(e1)==((size_t)e2) ?						\
-   _gen_result(1, __func__, __FILE__, __LINE__,"%s == %s",#e1,#e2) :	\
-   (_gen_result(0, __func__, __FILE__, __LINE__,			\
-		"%s == %s",#e1,#e2)) || (diag("Expected: %zd",(size_t)e2),diag("     Got: %zd",(size_t)e1),0)) /* diag is void; note commas. */
-
-#endif

ccan/ccan/base64/test/run.c

@@ -1,359 +0,0 @@
-/* Start of run.c test */
-#include "config.h"
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-
-#include <ccan/base64/base64.h>
-#include <ccan/tap/tap.h>
-
-#include <ccan/base64/base64.c>
-#include "moretap.h"
-
-static void * xmalloc(size_t size);
-
-/* not defined in terms of test_encode_using_maps so we cross
-   appropriate paths in library */
-#define test_encode(src,srclen,expected)			\
-	do {							\
-		size_t destlen;					\
-		char * dest;					\
-		destlen = base64_encoded_length(srclen);	\
-		destlen++; /* null termination */		\
-		dest = xmalloc(destlen);			\
-		ok1(base64_encode(dest,destlen,src,srclen) != -1);	\
-		is_str(dest,expected);					\
-		free(dest);						\
-	} while (0)
-
-#define test_encode_using_alphabet(alphastring,src,srclen,expected)	\
-	do {								\
-		size_t destlen;						\
-		char * dest;						\
-		base64_maps_t maps;				\
-		base64_init_maps(&maps,alphastring);		\
-		destlen = base64_encoded_length(srclen);		\
-		destlen++; /* null termination */		\
-		dest = xmalloc(destlen);				\
-		ok1(base64_encode_using_maps(&maps,dest,destlen,src,srclen) != -1); \
-		is_str(dest,expected);					\
-		free(dest);						\
-	} while (0)
-
-/* not defined in terms of test_decode_using_alphabet so we cross
-   appropriate paths in library */
-#define test_decode(src,srclen,expected,expectedlen)			\
-	do {								\
-		size_t destlen;						\
-		size_t bytes_used;					\
-		char * dest;						\
-		destlen = base64_decoded_length(srclen);		\
-		dest = xmalloc(destlen);				\
-		ok1((bytes_used = base64_decode(dest,destlen,src,srclen)) != -1); \
-		is_size_t(bytes_used,expectedlen);			\
-		is_mem(dest,expected,bytes_used);			\
-		free(dest);						\
-	} while (0)
-
-#define test_decode_using_alphabet(alphastring,src,srclen,expected,expectedlen) \
-	do {								\
-		size_t destlen;						\
-		size_t bytes_used;					\
-		char * dest;						\
-		base64_maps_t maps;				\
-									\
-		base64_init_maps(&maps,alphastring);		\
-		destlen = base64_decoded_length(srclen);		\
-		dest = xmalloc(destlen);				\
-		ok1((bytes_used = base64_decode_using_maps(&maps,dest,destlen,src,srclen)) != -1); \
-		is_size_t(bytes_used,expectedlen);			\
-		is_mem(dest,expected,bytes_used);			\
-		free(dest);						\
-	} while (0)
-
-#define check_bad_range_decode(stuff_to_test,stufflen)	\
-do {							\
-	char dest[10];							\
-	errno = 0;							\
-	is_size_t(base64_decode(dest,sizeof(dest),stuff_to_test,(size_t)stufflen), \
-		  (size_t)-1);						\
-	is_int(errno,EDOM);						\
-} while (0)
-
-int
-main(int argc, char *argv[])
-{
-	plan_tests(131);
-
-	is_size_t(base64_encoded_length(0),(size_t)0);
-	is_size_t(base64_encoded_length(1),(size_t)4);
-	is_size_t(base64_encoded_length(2),(size_t)4);
-	is_size_t(base64_encoded_length(3),(size_t)4);
-	is_size_t(base64_encoded_length(512),(size_t)684);
-
-	/* straight from page 11 of http://tools.ietf.org/html/rfc4648 */
-	test_encode("",0,"");
-	test_encode("f",1,"Zg==");
-	test_encode("fo",2,"Zm8=");
-
-	test_encode("foo",3,"Zm9v");
-	test_encode("foob",4,"Zm9vYg==");
-	test_encode("fooba",5,"Zm9vYmE=");
-	test_encode("foobar",6,"Zm9vYmFy");
-
-	/* a few more */
-	test_encode("foobarb",7,"Zm9vYmFyYg==");
-	test_encode("foobarba",8,"Zm9vYmFyYmE=");
-	test_encode("foobarbaz",9,"Zm9vYmFyYmF6");
-
-	test_encode("foobart",7,"Zm9vYmFydA==");
-
-	test_encode("abcdefghijklmnopqrstuvwxyz",26,"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=");
-	test_encode("\x05\x05\x01\x00\x07",5,"BQUBAAc=");
-
-	test_encode("FOO",3,"Rk9P");
-	test_encode("Z",1,"Wg==");
-
-	/* decode testing */
-
-	test_decode("",0,"",0);
-	test_decode("Zg==",4,"f",1);
-	test_decode("Zm8=",4,"fo",2);
-	test_decode("Zm9v",4,"foo",3);
-	test_decode("Zm9vYg==",8,"foob",4);
-	test_decode("Zm9vYmE=",8,"fooba",5);
-	test_decode("Zm9vYmFy",8,"foobar",6);
-	test_decode("Zm9vYmFyYg==",12,"foobarb",7);
-	test_decode("Zm9vYmFyYmE=",12,"foobarba",8);
-	test_decode("Zm9vYmFyYmF6",12,"foobarbaz",9);
-
-	test_decode("Rk9P",4,"FOO",3);
-
-	test_decode("Wg==",4,"Z",1);
-	test_decode("AA==",4,"\0",1);
-	test_decode("AAA=",4,"\0\0",2);
-
-	{
-		const char *binary = "\x01\x00\x03";
-		const size_t binarylen = 3;
-
-		char * decoded;
-		char * encoded;
-		size_t encoded_len;
-		size_t decoded_len;
-		size_t decoded_space_required;
-
-		size_t encoded_space_required = base64_encoded_length(binarylen);
-		encoded_space_required++; /* null termination */
-		encoded = xmalloc(encoded_space_required);
-		encoded_len = base64_encode(encoded,encoded_space_required,binary,binarylen);
-		is_mem(encoded,"AQAD",encoded_len);
-
-		decoded_space_required = base64_decoded_length(encoded_len);
-		decoded = xmalloc(decoded_space_required);
-		decoded_len = base64_decode(decoded,decoded_space_required,encoded,encoded_len);
-		is_size_t(decoded_len,binarylen);
-		is_mem(binary,decoded,decoded_len);
-	}
-
-	/* some expected encode failures: */
-	{
-		size_t destlen = 1;
-		char dest[destlen];
-		errno = 0;
-		is_size_t(base64_encode(dest,destlen,"A",1),(size_t)-1);
-		is_int(errno,EOVERFLOW);
-	}
-
-	/* some expected decode failures: */
-	{
-		base64_maps_t maps;
-		const char * src = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-		base64_init_maps(&maps,src);
-
-		is_int(sixbit_from_b64(&maps,'\xfe'),(signed char)-1);
-		is_int(errno,EDOM);
-	}
-	{
-		size_t destlen = 10;
-		char dest[destlen];
-		errno = 0;
-		is_size_t(base64_decode(dest,destlen,"A",1),(size_t)-1);
-		is_int(errno,EINVAL);
-	}
-	{
-		size_t destlen = 1;
-		char dest[destlen];
-		errno = 0;
-		is_size_t(base64_decode(dest,destlen,"A",1),(size_t)-1);
-		is_int(errno,EOVERFLOW);
-	}
-	{
-		/* (char)1 is not a valid base64 character: */
-		check_bad_range_decode("A\x01",2);
-		/* (char)255 is not a valid base64 character: (char is signed on most platforms, so this is actually < 0 */
-		check_bad_range_decode("\xff""A",2);
-		check_bad_range_decode("A\xff",2);
-		check_bad_range_decode("AA\xff",3);
-		check_bad_range_decode("A\xff""A",3);
-		check_bad_range_decode("\xff""AA",3);
-		check_bad_range_decode("AAA\xff",4);
-		check_bad_range_decode("\xff\x41\x41\x41\x41",5);
-		check_bad_range_decode("A\xff\x41\x41\x41\x41",6);
-		check_bad_range_decode("AA\xff\x41\x41\x41\x41",7);
-		check_bad_range_decode("AAA\xff\x41\x41\x41\x41",8);
-	}
-	/* trigger some failures in the sixbit-to-b64 encoder: */
-	/* this function now aborts rather than returning -1/setting errno */
-	/* { */
-	/* 	is_int(sixbit_to_b64(base64_maps_rfc4648,'\x70'),(char)-1); */
-	/* 	is_int(sixbit_to_b64(base64_maps_rfc4648,'\xff'),(char)-1); */
-	/* } */
-	/* following tests all of the mapping from b64 chars to 6-bit values: */
-	test_decode("//+FwHRSRIsFU2IhAEGD+AMPhOA=",28,"\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0",20);
-	test_encode("\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0",20,"//+FwHRSRIsFU2IhAEGD+AMPhOA=");
-
-
-	/* check the null-padding stuff */
-	{
-		size_t destlen = 8;
-		char dest[destlen];
-		memset(dest,'\1',sizeof(dest));
-		is_size_t(base64_encode(dest,destlen,"A",1),(size_t)4);
-		is_mem(&dest[4],"\0\0\0\0",4);
-	}
-	{
-		size_t destlen = 3;
-		char dest[destlen];
-		memset(dest,'\1',sizeof(dest));
-		is_size_t(base64_decode(dest,destlen,"Wg==",4), 1);
-		is_mem(&dest[1],"\0",2);
-	}
-
-	/* test encoding using different alphabets */
-	{
-		char alphabet_fs_safe[64];
-		memcpy(alphabet_fs_safe,base64_maps_rfc4648.encode_map,sizeof(alphabet_fs_safe));
-		alphabet_fs_safe[62] = '-';
-		alphabet_fs_safe[63] = '_';
-		test_encode_using_alphabet(alphabet_fs_safe,"\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0",20,"__-FwHRSRIsFU2IhAEGD-AMPhOA=");
-	}
-
-	/* test decoding using different alphabets */
-	{
-		char alphabet_fs_safe[64];
-		#define src "__-FwHRSRIsFU2IhAEGD-AMPhOA="
-		#define expected "\xff\xff\x85\xc0\x74\x52\x44\x8b\x05\x53\x62\x21\x00\x41\x83\xf8\x03\x0f\x84\xe0"
-
-		memcpy(alphabet_fs_safe,base64_maps_rfc4648.encode_map,sizeof(alphabet_fs_safe));
-		alphabet_fs_safe[62] = '-';
-		alphabet_fs_safe[63] = '_';
-
-		test_decode_using_alphabet(alphabet_fs_safe,src,strlen(src),expected,20);
-		#undef src
-		#undef expected
-	}
-
-	/* explicitly test the non-maps encode_triplet and
-	   encode_tail functions */
-	{
-		size_t destlen = 4;
-		char dest[destlen];
-		const char *src = "AB\04";
-		memset(dest,'\1',sizeof(dest));
-		base64_encode_triplet(dest,src);
-		is_mem(dest,"QUIE",sizeof(dest));
-	}
-	{
-		size_t destlen = 4;
-		char dest[destlen];
-		const char *src = "A";
-		memset(dest,'\1',sizeof(dest));
-		base64_encode_tail(dest,src,strlen(src));
-		is_mem(dest,"QQ==",sizeof(dest));
-	}
-
-	/* test the alphabet inversion */
-	{
-		base64_maps_t dest;
-		const char expected_inverse[] =
-			"\xff\xff\xff\xff\xff" /* 0 */
-			"\xff\xff\xff\xff\xff" /* 5 */
-			"\xff\xff\xff\xff\xff" /* 10 */
-			"\xff\xff\xff\xff\xff" /* 15 */
-			"\xff\xff\xff\xff\xff" /* 20 */
-			"\xff\xff\xff\xff\xff" /* 25 */
-			"\xff\xff\xff\xff\xff" /* 30 */
-			"\xff\xff\xff\xff\xff" /* 35 */
-			"\xff\xff\xff\x3e\xff" /* 40 */
-			"\xff\xff\x3f\x34\x35" /* 45 - */
-			"\x36\x37\x38\x39\x3a" /* 50 */
-			"\x3b\x3c\x3d\xff\xff" /* 55 */
-			"\xff\xff\xff\xff\xff" /* 60 */
-			"\x00\x01\x02\x03\x04" /* 65 A */
-			"\x05\x06\x07\x08\x09" /* 70 */
-			"\x0a\x0b\x0c\x0d\x0e" /* 75 */
-			"\x0f\x10\x11\x12\x13" /* 80 */
-			"\x14\x15\x16\x17\x18" /* 85 */
-			"\x19\xff\xff\xff\xff" /* 90 */
-			"\xff\xff\x1a\x1b\x1c" /* 95 _ */
-			"\x1d\x1e\x1f\x20\x21" /* 100 */
-			"\x22\x23\x24\x25\x26" /* 105 */
-			"\x27\x28\x29\x2a\x2b" /* 110 */
-			"\x2c\x2d\x2e\x2f\x30" /* 115 */
-			"\x31\x32\x33\xff\xff" /* 120 */
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 125 */
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 155 */
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 185 */
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 215 */
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-			"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 245 */
-			;
-		const char * src = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-		base64_init_maps(&dest, src);
-		is_mem((const char *)dest.decode_map, expected_inverse, 256);
-		ok1(base64_char_in_alphabet(&dest,'A'));
-		ok1(!base64_char_in_alphabet(&dest,'\n'));
-	}
-
-	/* explicitly test the non-alpha decode_tail and decode_quartet */
-	{
-		char dest[4];
-		const char *src = "QQ==";
-		const char * expected = "A";
-		memset(dest, '%', sizeof(dest));
-		base64_decode_tail(dest,src,4);
-		is_mem(dest, expected, 1);
-	}
-	{
-		char dest[4];
-		const char *src = "Zm9v";
-		const char * expected = "foo";
-		memset(dest, '%', sizeof(dest));
-		base64_decode_quartet(dest,src);
-		is_mem(dest, expected, 1);
-	}
-
-	exit(exit_status());
-}
-
-static void * xmalloc(size_t size)
-{
-	char * ret;
-	ret = malloc(size);
-	if (ret == NULL) {
-		perror("malloc");
-		abort();
-	}
-	return ret;
-}
-
-/* End of run.c test */

ccan/ccan/bitmap/bitmap.c

@@ -9,7 +9,7 @@
 #define BIT_ALIGN_DOWN(n)	((n) & ~(BITMAP_WORD_BITS - 1))
 #define BIT_ALIGN_UP(n)		BIT_ALIGN_DOWN((n) + BITMAP_WORD_BITS - 1)
 
-void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m)
+void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
 	unsigned long am = BIT_ALIGN_DOWN(m);
@@ -19,22 +19,22 @@ void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m)
 	assert(m >= n);
 
 	if (am < an) {
-		BITMAP_WORD(b, n) &= ~bitmap_bswap(headmask & tailmask);
+		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask & tailmask);
 		return;
 	}
 
 	if (an > n)
-		BITMAP_WORD(b, n) &= ~bitmap_bswap(headmask);
+		BITMAP_WORD(bitmap, n) &= ~bitmap_bswap(headmask);
 
 	if (am > an)
-		memset(&BITMAP_WORD(b, an), 0,
+		memset(&BITMAP_WORD(bitmap, an), 0,
 		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
 
 	if (m > am)
-		BITMAP_WORD(b, m) &= ~bitmap_bswap(tailmask);
+		BITMAP_WORD(bitmap, m) &= ~bitmap_bswap(tailmask);
 }
 
-void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m)
+void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
 	unsigned long am = BIT_ALIGN_DOWN(m);
@@ -44,19 +44,19 @@ void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m)
 	assert(m >= n);
 
 	if (am < an) {
-		BITMAP_WORD(b, n) |= bitmap_bswap(headmask & tailmask);
+		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask & tailmask);
 		return;
 	}
 
 	if (an > n)
-		BITMAP_WORD(b, n) |= bitmap_bswap(headmask);
+		BITMAP_WORD(bitmap, n) |= bitmap_bswap(headmask);
 
 	if (am > an)
-		memset(&BITMAP_WORD(b, an), 0xff,
+		memset(&BITMAP_WORD(bitmap, an), 0xff,
 		       (am - an) / BITMAP_WORD_BITS * sizeof(bitmap_word));
 
 	if (m > am)
-		BITMAP_WORD(b, m) |= bitmap_bswap(tailmask);
+		BITMAP_WORD(bitmap, m) |= bitmap_bswap(tailmask);
 }
 
 static int bitmap_clz(bitmap_word w)
@@ -76,7 +76,7 @@ static int bitmap_clz(bitmap_word w)
 #endif
 }
 
-unsigned long bitmap_ffs(const bitmap *b,
+unsigned long bitmap_ffs(const bitmap *bitmap,
 			 unsigned long n, unsigned long m)
 {
 	unsigned long an = BIT_ALIGN_UP(n);
@@ -87,7 +87,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	assert(m >= n);
 
 	if (am < an) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, n));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
 
 		w &= (headmask & tailmask);
 
@@ -95,7 +95,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	if (an > n) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, n));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, n));
 
 		w &= headmask;
 
@@ -104,7 +104,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	while (an < am) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, an));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, an));
 
 		if (w)
 			return an + bitmap_clz(w);
@@ -113,7 +113,7 @@ unsigned long bitmap_ffs(const bitmap *b,
 	}
 
 	if (m > am) {
-		bitmap_word w = bitmap_bswap(BITMAP_WORD(b, m));
+		bitmap_word w = bitmap_bswap(BITMAP_WORD(bitmap, m));
 
 		w &= tailmask;
 

ccan/ccan/bitmap/bitmap.h

@@ -58,37 +58,37 @@ static inline bitmap_word bitmap_bswap(bitmap_word w)
 #define BITMAP_TAIL(_bm, _nbits) \
 	(BITMAP_TAILWORD(_bm, _nbits) & BITMAP_TAILBITS(_nbits))
 
-static inline void bitmap_set_bit(bitmap *b, unsigned long n)
+static inline void bitmap_set_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) |= BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) |= BITMAP_WORDBIT(n);
 }
 
-static inline void bitmap_clear_bit(bitmap *b, unsigned long n)
+static inline void bitmap_clear_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) &= ~BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) &= ~BITMAP_WORDBIT(n);
 }
 
-static inline void bitmap_change_bit(bitmap *b, unsigned long n)
+static inline void bitmap_change_bit(bitmap *bitmap, unsigned long n)
 {
-	BITMAP_WORD(b, n) ^= BITMAP_WORDBIT(n);
+	BITMAP_WORD(bitmap, n) ^= BITMAP_WORDBIT(n);
 }
 
-static inline bool bitmap_test_bit(const bitmap *b, unsigned long n)
+static inline bool bitmap_test_bit(const bitmap *bitmap, unsigned long n)
 {
-	return !!(BITMAP_WORD(b, n) & BITMAP_WORDBIT(n));
+	return !!(BITMAP_WORD(bitmap, n) & BITMAP_WORDBIT(n));
 }
 
-void bitmap_zero_range(bitmap *b, unsigned long n, unsigned long m);
-void bitmap_fill_range(bitmap *b, unsigned long n, unsigned long m);
+void bitmap_zero_range(bitmap *bitmap, unsigned long n, unsigned long m);
+void bitmap_fill_range(bitmap *bitmap, unsigned long n, unsigned long m);
 
-static inline void bitmap_zero(bitmap *b, unsigned long nbits)
+static inline void bitmap_zero(bitmap *bitmap, unsigned long nbits)
 {
-	memset(b, 0, bitmap_sizeof(nbits));
+	memset(bitmap, 0, bitmap_sizeof(nbits));
 }
 
-static inline void bitmap_fill(bitmap *b, unsigned long nbits)
+static inline void bitmap_fill(bitmap *bitmap, unsigned long nbits)
 {
-	memset(b, 0xff, bitmap_sizeof(nbits));
+	memset(bitmap, 0xff, bitmap_sizeof(nbits));
 }
 
 static inline void bitmap_copy(bitmap *dst, const bitmap *src,
@@ -161,36 +161,37 @@ static inline bool bitmap_subset(const bitmap *src1, const bitmap *src2,
 	return true;
 }
 
-static inline bool bitmap_full(const bitmap *b, unsigned long nbits)
+static inline bool bitmap_full(const bitmap *bitmap, unsigned long nbits)
 {
 	unsigned long i;
 
 	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
-		if (b[i].w != -1UL)
+		if (bitmap[i].w != -1UL)
 			return false;
 	}
 	if (BITMAP_HASTAIL(nbits) &&
-	    (BITMAP_TAIL(b, nbits) != BITMAP_TAILBITS(nbits)))
+	    (BITMAP_TAIL(bitmap, nbits) != BITMAP_TAILBITS(nbits)))
 		return false;
 
 	return true;
 }
 
-static inline bool bitmap_empty(const bitmap *b, unsigned long nbits)
+static inline bool bitmap_empty(const bitmap *bitmap, unsigned long nbits)
 {
 	unsigned long i;
 
 	for (i = 0; i < BITMAP_HEADWORDS(nbits); i++) {
-		if (b[i].w != 0)
+		if (bitmap[i].w != 0)
 			return false;
 	}
-	if (BITMAP_HASTAIL(nbits) && (BITMAP_TAIL(b, nbits) != 0))
+	if (BITMAP_HASTAIL(nbits) && (BITMAP_TAIL(bitmap, nbits) != 0))
 		return false;
 
 	return true;
 }
 
-unsigned long bitmap_ffs(const bitmap *b, unsigned long n, unsigned long m);
+unsigned long bitmap_ffs(const bitmap *bitmap,
+			 unsigned long n, unsigned long m);
 
 /*
  * Allocation functions
@@ -220,26 +221,26 @@ static inline bitmap *bitmap_alloc1(unsigned long nbits)
 	return bitmap;
 }
 
-static inline bitmap *bitmap_realloc0(bitmap *b,
+static inline bitmap *bitmap_realloc0(bitmap *bitmap,
 				      unsigned long obits, unsigned long nbits)
 {
-	b = realloc(b, bitmap_sizeof(nbits));
+	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
 
-	if ((nbits > obits) && b)
-		bitmap_zero_range(b, obits, nbits);
+	if ((nbits > obits) && bitmap)
+		bitmap_zero_range(bitmap, obits, nbits);
 
-	return b;
+	return bitmap;
 }
 
-static inline bitmap *bitmap_realloc1(bitmap *b,
+static inline bitmap *bitmap_realloc1(bitmap *bitmap,
 				      unsigned long obits, unsigned long nbits)
 {
-	b = realloc(b, bitmap_sizeof(nbits));
+	bitmap = realloc(bitmap, bitmap_sizeof(nbits));
 
-	if ((nbits > obits) && b)
-		bitmap_fill_range(b, obits, nbits);
+	if ((nbits > obits) && bitmap)
+		bitmap_fill_range(bitmap, obits, nbits);
 
-	return b;
+	return bitmap;
 }
 
 #endif /* CCAN_BITMAP_H_ */

ccan/ccan/bitops/test/run.c

@@ -10,7 +10,7 @@ int main(void)
 	plan_tests(68 + 6 * (31 + 63));
 
 	for (i = 0; i < 32; i++)
-		ok1(bitops_ffs32(1U << i) == i+1);
+		ok1(bitops_ffs32(1 << i) == i+1);
 	ok1(bitops_ffs32(0) == 0);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_ffs64((uint64_t)1 << i) == i+1);
@@ -25,19 +25,19 @@ int main(void)
 	ok1(bitops_ffs64(0) == 0);
 
 	for (i = 0; i < 32; i++)
-		ok1(bitops_clz32(1U << i) == 31 - i);
+		ok1(bitops_clz32(1 << i) == 31 - i);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_clz64((uint64_t)1 << i) == 63 - i);
 
 	/* Lower bits don't effect results */
 	for (i = 0; i < 32; i++)
-		ok1(bitops_clz32((1U << i) + (1U << i)-1) == 31 - i);
+		ok1(bitops_clz32((1 << i) + (1 << i)-1) == 31 - i);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_clz64(((uint64_t)1 << i) + ((uint64_t)1 << i)-1)
 		    == 63 - i);
 
 	for (i = 0; i < 32; i++)
-		ok1(bitops_ctz32(1U << i) == i);
+		ok1(bitops_ctz32(1 << i) == i);
 	for (i = 0; i < 64; i++)
 		ok1(bitops_ctz64((uint64_t)1 << i) == i);
 

ccan/ccan/closefrom/LICENSE

@@ -1 +0,0 @@
-../../licenses/CC0

ccan/ccan/closefrom/_info

@@ -1,69 +0,0 @@
-#include "config.h"
-#include <stdio.h>
-#include <string.h>
-
-/**
- * closefrom - close all fds starting from specified fd.
- *
- * This code is an example of what to do in a child process to
- * ensure that none of the (possibly sensitive) file descriptors
- * in the parent remain in the child process.
- *
- * License: CC0 (Public domain)
- * Author: ZmnSCPxj jxPCSnmZ <ZmnSCPxj@protonmail.com>
- *
- * Example:
- * #include <ccan/closefrom/closefrom.h>
- * #include <ccan/err/err.h>
- * #include <stdio.h>
- * #include <sys/resource.h>
- * #include <sys/time.h>
- * #include <sys/types.h>
- * #include <sys/wait.h>
- * #include <unistd.h>
- *
- * int main(int argc, char **argv)
- * {
- *	pid_t child;
- *
- *	// If being emulated, then we might end up
- *	// looping over a large _SC_OPEN_MAX
- *	// (Some systems have it as INT_MAX!)
- *	// If so, closefrom_limit will lower this limit
- *	// to a value you specify, or if given 0 will
- *	// limit to 4096.
- *	// Call this as early as possible.
- *	closefrom_limit(0);
- *
- *	// If we limited, we can query this so we can
- *	// print it in debug logs or something.
- *	if (closefrom_may_be_slow())
- *		printf("we limited ourselves to 4096 fds.\n");
- *
- *	child = fork();
- *	if (child < 0)
- *		err(1, "Forking");
- *	if (child == 0) {
- *		closefrom(STDERR_FILENO + 1);
- *		// Insert your *whatever* code here.
- *		_exit(0);
- *	}
- *
- *	waitpid(child, NULL, 0);
- *
- *	return 0;
- * }
- *
- */
-int main(int argc, char *argv[])
-{
-	/* Expect exactly one argument */
-	if (argc != 2)
-		return 1;
-
-	if (strcmp(argv[1], "depends") == 0) {
-		return 0;
-	}
-
-	return 1;
-}

ccan/ccan/closefrom/closefrom.c

@@ -1,225 +0,0 @@
-/* CC0 license (public domain) - see LICENSE file for details */
-#include <ccan/closefrom/closefrom.h>
-#include <dirent.h>
-#include <errno.h>
-#include <limits.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <sys/time.h>
-#include <sys/resource.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-/* See also:
- * https://stackoverflow.com/a/918469
- *
- * The implementation below is not exhaustive of all the suggested above.
- */
-
-#if !HAVE_CLOSEFROM
-
-/* IBM AIX.
- * https://www.ibm.com/docs/en/aix/7.2?topic=f-fcntl-dup-dup2-subroutine
- */
-#if HAVE_F_CLOSEM
-
-#include <fcntl.h>
-
-void closefrom(int fromfd)
-{
-	(void) fcntl(fromfd, F_CLOSEM, 0);
-}
-
-bool closefrom_may_be_slow(void)
-{
-	return false;
-}
-
-#else /* !HAVE_F_CLOSEM */
-
-#if HAVE_NR_CLOSE_RANGE
-#include <sys/syscall.h>
-#endif
-
-#define PROC_PID_FD_LEN \
-	( 6  /* /proc/ */ \
-	+ 20 /* 64-bit $PID */ \
-	+ 3  /* /fd */ \
-	+ 1  /* NUL */ \
-	)
-
-static bool can_get_maxfd(void)
-{
-#if HAVE_F_MAXFD
-	int res = fcntl(0, F_MAXFD);
-	if (res < 0)
-		return false;
-	else
-		return true;
-#else
-	return false;
-#endif
-}
-
-/* Linux >= 5.9 */
-static bool can_close_range(void)
-{
-#if HAVE_NR_CLOSE_RANGE
-	int res = syscall(__NR_close_range, INT_MAX, INT_MAX, 0);
-	if (res < 0)
-		return false;
-	return true;
-#else
-	return false;
-#endif
-}
-
-/* On Linux, Solaris, AIX, Cygwin, and NetBSD.  */
-static bool can_open_proc_pid_fd(void)
-{
-	char dnam[PROC_PID_FD_LEN];
-	DIR *dir;
-
-	sprintf(dnam, "/proc/%ld/fd", (long) getpid());
-	dir = opendir(dnam);
-	if (!dir)
-		return false;
-	closedir(dir);
-	return true;
-}
-
-/* On FreeBSD and MacOS.  */
-static bool can_open_dev_fd(void)
-{
-	DIR *dir;
-	dir = opendir("/dev/fd");
-	if (!dir)
-		return false;
-	closedir(dir);
-	return true;
-}
-
-bool closefrom_may_be_slow(void)
-{
-	if (can_get_maxfd())
-		return false;
-	else if (can_close_range())
-		return false;
-	else if (can_open_proc_pid_fd())
-		return false;
-	else if (can_open_dev_fd())
-		return false;
-	else
-		return true;
-}
-
-/* It is possible that we run out of available file descriptors.
- * However, if we are going to close anyway, we could just try
- * closing file descriptors until we reach maxfd.
- */
-static
-DIR *try_opendir(const char *dnam, int *fromfd, int maxfd)
-{
-	DIR *dir;
-
-	do {
-		dir = opendir(dnam);
-		if (!dir && (errno == ENFILE || errno == EMFILE)) {
-			if (*fromfd < maxfd)
-				close((*fromfd)++);
-			else
-				break;
-		}
-	} while (!dir && (errno == ENFILE || errno == EMFILE));
-
-	return dir;
-}
-
-void closefrom(int fromfd)
-{
-	int saved_errno = errno;
-
-	int res;
-	int maxfd;
-
-	char dnam[PROC_PID_FD_LEN];
-	DIR *dir;
-	struct dirent *entry;
-
-	(void) res;
-
-	if (fromfd < 0)
-		goto quit;
-
-#if HAVE_NR_CLOSE_RANGE
-	res = syscall(__NR_close_range, fromfd, INT_MAX, 0);
-	if (res == 0)
-		goto quit;
-#endif
-
-	maxfd = sysconf(_SC_OPEN_MAX);
-
-	sprintf(dnam, "/proc/%ld/fd", (long) getpid());
-	dir = try_opendir(dnam, &fromfd, maxfd);
-	if (!dir)
-		dir = try_opendir("/dev/fd", &fromfd, maxfd);
-
-	if (dir) {
-		while ((entry = readdir(dir))) {
-			long fd;
-			char *endp;
-
-			fd = strtol(entry->d_name, &endp, 10);
-			if (entry->d_name != endp && *endp == '\0' &&
-			    fd >= 0 && fd < INT_MAX && fd >= fromfd &&
-			    fd != dirfd(dir) )
-				close(fd);
-		}
-		closedir(dir);
-		goto quit;
-	}
-
-#if HAVE_F_MAXFD
-	res = fcntl(0, F_MAXFD);
-	if (res >= 0)
-		maxfd = res + 1;
-#endif
-
-	/* Fallback.  */
-	for (; fromfd < maxfd; ++fromfd)
-		close(fromfd);
-
-quit:
-	errno = saved_errno;
-}
-
-#endif /* !HAVE_F_CLOSEM */
-
-void closefrom_limit(unsigned int arg_limit)
-{
-	rlim_t limit = (rlim_t) arg_limit;
-
-	struct rlimit nofile;
-
-	if (!closefrom_may_be_slow())
-		return;
-
-	if (limit == 0)
-		limit = 4096;
-
-	getrlimit(RLIMIT_NOFILE, &nofile);
-
-	/* Respect the max limit.
-	 * If we are not running as root then we cannot raise
-	 * it, but we *can* lower the max limit.
-	 */
-	if (nofile.rlim_max != RLIM_INFINITY && limit > nofile.rlim_max)
-		limit = nofile.rlim_max;
-
-	nofile.rlim_cur = limit;
-	nofile.rlim_max = limit;
-
-	setrlimit(RLIMIT_NOFILE, &nofile);
-}
-
-#endif /* !HAVE_CLOSEFROM */

ccan/ccan/closefrom/closefrom.h

@@ -1,81 +0,0 @@
-/* CC0 license (public domain) - see LICENSE file for details */
-#ifndef CCAN_CLOSEFROM_H
-#define CCAN_CLOSEFROM_H
-#include "config.h"
-#include <stdbool.h>
-
-#if HAVE_CLOSEFROM
-/* BSD.  */
-#include <unistd.h>
-/* Solaris.  */
-#include <stdlib.h>
-
-static inline
-bool closefrom_may_be_slow(void)
-{
-	return 0;
-}
-
-static inline
-void closefrom_limit(unsigned int limit)
-{
-}
-
-#else /* !HAVE_CLOSEFROM */
-
-/**
- * closefrom - Close all open file descriptors, starting
- * at fromfd onwards.
- * @fromfd: the first fd to close; it and all higher file descriptors
- * will be closed.
- *
- * This is not multithread-safe: other threads in the same process
- * may or may not open new file descriptors in parallel to this call.
- * However, the expected use-case is that this will be called in a
- * child process just after fork(), meaning the child process is still
- * single-threaded.
- */
-void closefrom_(int fromfd);
-/* In case the standard library has it, but declared in some
- * *other* header we do not know of yet, we use closefrom_ in
- * the actual name the linker sees.
- */
-#define closefrom closefrom_
-
-/**
- * closefrom_may_be_slow - check if the closefrom() function could
- * potentially take a long time.
- *
- * The return value is true if closefrom() is emulated by
- * looping from fromfd to sysconf(_SC_OPEN_MAX), which can be
- * very large (possibly even INT_MAX on some systems).
- * If so, you might want to use setrlimit to limit _SC_OPEN_MAX.
- * If this returns false, then closefrom is efficient and you do not
- * need to limit the number of file descriptors.
- *
- * You can use closefrom_limit to perform the limiting based on
- * closefrom_may_be_slow.
- * This API is exposed in case you want to output to debug logs or
- * something similar.
- */
-bool closefrom_may_be_slow(void);
-
-/**
- * closefrom_limit - If closefrom_may_be_slow(), lower the limit on
- * the number of file descriptors we keep open, to prevent closefrom
- * from being *too* slow.
- * @limit: 0 to use a reasonable default of 4096, or non-zero for the
- * limit you prefer.
- *
- * This function does nothing if closefrom_may_be_slow() return false.
- *
- * This function only *lowers* the limit from the hard limit set by
- * root before running this program.
- * If the limit is higher than the hard limit, then the hard limit is
- * respected.
- */
-void closefrom_limit(unsigned int limit);
-
-#endif /* !HAVE_CLOSEFROM */
-
-#endif /* CCAN_CLOSEFROM_H */

ccan/ccan/closefrom/test/run.c

@@ -1,192 +0,0 @@
-#include <ccan/closefrom/closefrom.h>
-/* Include the C files directly. */
-#include <ccan/closefrom/closefrom.c>
-#include <ccan/tap/tap.h>
-#include <errno.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <unistd.h>
-
-/* Open a pipe, do closefrom, check pipe no longer works.   */
-static
-int pipe_close(void)
-{
-	int fds[2];
-	ssize_t wres;
-
-	char buf = '\0';
-
-	if (pipe(fds) < 0)
-		return 0;
-
-	/* Writing to the write end should succeed, the
-	 * pipe is working.  */
-	do {
-		wres = write(fds[1], &buf, 1);
-	} while ((wres < 0) && (errno == EINTR));
-	if (wres < 0)
-		return 0;
-
-	closefrom(STDERR_FILENO + 1);
-
-	/* Writing to the write end should fail because
-	 * everything should be closed.  */
-	do {
-		wres = write(fds[1], &buf, 1);
-	} while ((wres < 0) && (errno == EINTR));
-
-	return (wres < 0) && (errno == EBADF);
-}
-
-/* Open a pipe, fork, do closefrom in child, read pipe from parent,
- * parent should see EOF.
- */
-static
-int fork_close(void)
-{
-	int fds[2];
-	pid_t child;
-
-	char buf;
-	ssize_t rres;
-
-	if (pipe(fds) < 0)
-		return 0;
-
-	child = fork();
-	if (child < 0)
-		return 0;
-
-	if (child == 0) {
-		/* Child.  */
-		closefrom(STDERR_FILENO + 1);
-		_exit(0);
-	} else {
-		/* Parent.  */
-
-		/* Close write end of pipe.  */
-		close(fds[1]);
-
-		do {
-			rres = read(fds[0], &buf, 1);
-		} while ((rres < 0) && (errno == EINTR));
-
-		/* Should have seen EOF.  */
-		if (rres != 0)
-			return 0;
-
-		/* Clean up.  */
-		waitpid(child, NULL, 0);
-		closefrom(STDERR_FILENO + 1);
-	}
-
-	return 1;
-}
-/* Open a pipe, fork, in child set the write end to fd #3,
- * in parent set the read end to fd #3, send a byte from
- * child to parent, check.
- */
-static
-int fork_communicate(void)
-{
-	int fds[2];
-	pid_t child;
-
-	char wbuf = 42;
-	char rbuf;
-	ssize_t rres;
-	ssize_t wres;
-
-	int status;
-
-	if (pipe(fds) < 0)
-		return 0;
-
-	child = fork();
-	if (child < 0)
-		return 0;
-
-	if (child == 0) {
-		/* Child.  */
-
-		/* Move write end to fd #3.  */
-		if (fds[1] != 3) {
-			if (dup2(fds[1], 3) < 0)
-				_exit(127);
-			close(fds[1]);
-			fds[1] = 3;
-		}
-
-		closefrom(4);
-
-		do {
-			wres = write(fds[1], &wbuf, 1);
-		} while ((wres < 0) && (errno == EINTR));
-		if (wres < 0)
-			_exit(127);
-
-		_exit(0);
-	} else {
-		/* Parent.  */
-
-		/* Move read end to fd #3.  */
-		if (fds[0] != 3) {
-			if (dup2(fds[0], 3) < 0)
-				return 0;
-			close(fds[0]);
-			fds[0] = 3;
-		}
-
-		closefrom(4);
-
-		/* Wait for child to finish.  */
-		waitpid(child, &status, 0);
-		if (!WIFEXITED(status))
-			return 0;
-		if (WEXITSTATUS(status) != 0)
-			return 0;
-
-		/* Read 1 byte.  */
-		do {
-			rres = read(fds[0], &rbuf, 1);
-		} while ((rres < 0) && (errno == EINTR));
-		if (rres < 0)
-			return 0;
-		if (rres != 1)
-			return 0;
-		/* Should get same byte as what was sent.  */
-		if (rbuf != wbuf)
-			return 0;
-
-		/* Next attempt to read should EOF.  */
-		do {
-			rres = read(fds[0], &rbuf, 1);
-		} while ((rres < 0) && (errno == EINTR));
-		if (rres < 0)
-			return 0;
-		/* Should EOF.  */
-		if (rres != 0)
-			return 0;
-
-	}
-
-	/* Clean up.  */
-	close(fds[0]);
-	return 1;
-}
-
-int main(void)
-{
-	/* Limit closefrom.  */
-	closefrom_limit(0);
-
-	/* This is how many tests you plan to run */
-	plan_tests(3);
-
-	ok1(pipe_close());
-	ok1(fork_close());
-	ok1(fork_communicate());
-
-	/* This exits depending on whether all tests passed */
-	return exit_status();
-}

ccan/ccan/compiler/compiler.h

@@ -271,19 +271,6 @@
 #define NON_NULL_ARGS(...)
 #endif
 
-#if HAVE_ATTRIBUTE_RETURNS_NONNULL
-/**
- * RETURNS_NONNULL - specify that this function cannot return NULL.
- *
- * Mainly an optimization opportunity, but can also suppress warnings.
- *
- * Example:
- * RETURNS_NONNULL char *my_copy(char *buf);
- */
-#define RETURNS_NONNULL __attribute__((__returns_nonnull__))
-#else
-#define RETURNS_NONNULL
-#endif
 
 #if HAVE_ATTRIBUTE_SENTINEL
 /**

ccan/ccan/crc32c/crc32c.c

@@ -72,8 +72,7 @@ static inline uint32_t gf2_matrix_times(uint32_t *mat, uint32_t vec) {
 /* Multiply a matrix by itself over GF(2).  Both mat and square must have 32
    rows. */
 static inline void gf2_matrix_square(uint32_t *square, uint32_t *mat) {
-    unsigned n;
-    for (n = 0; n < 32; n++)
+    for (unsigned n = 0; n < 32; n++)
         square[n] = gf2_matrix_times(mat, mat[n]);
 }
 
@@ -88,8 +87,7 @@ static void crc32c_zeros_op(uint32_t *even, size_t len) {
     /* put operator for one zero bit in odd */
     odd[0] = POLY;              /* CRC-32C polynomial */
     uint32_t row = 1;
-    unsigned n;
-    for (n = 1; n < 32; n++) {
+    for (unsigned n = 1; n < 32; n++) {
         odd[n] = row;
         row <<= 1;
     }
@@ -113,7 +111,7 @@ static void crc32c_zeros_op(uint32_t *even, size_t len) {
     } while (len);
 
     /* answer ended up in odd -- copy to even */
-    for (n = 0; n < 32; n++)
+    for (unsigned n = 0; n < 32; n++)
         even[n] = odd[n];
 }
 
@@ -123,8 +121,7 @@ static void crc32c_zeros(uint32_t zeros[][256], size_t len) {
     uint32_t op[32];
 
     crc32c_zeros_op(op, len);
-    unsigned n;
-    for (n = 0; n < 256; n++) {
+    for (unsigned n = 0; n < 256; n++) {
         zeros[0][n] = gf2_matrix_times(op, n);
         zeros[1][n] = gf2_matrix_times(op, n << 8);
         zeros[2][n] = gf2_matrix_times(op, n << 16);
@@ -268,8 +265,7 @@ uint32_t crc32c(uint32_t crc, void const *buf, size_t len) {
 static bool crc32c_once_little;
 static uint32_t crc32c_table_little[8][256];
 static void crc32c_init_sw_little(void) {
-    unsigned n;
-    for (n = 0; n < 256; n++) {
+    for (unsigned n = 0; n < 256; n++) {
         uint32_t crc = n;
         crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
         crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
@@ -281,10 +277,9 @@ static void crc32c_init_sw_little(void) {
         crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
         crc32c_table_little[0][n] = crc;
     }
-    for (n = 0; n < 256; n++) {
+    for (unsigned n = 0; n < 256; n++) {
         uint32_t crc = crc32c_table_little[0][n];
-        unsigned k;
-        for (k = 1; k < 8; k++) {
+        for (unsigned k = 1; k < 8; k++) {
             crc = crc32c_table_little[0][crc & 0xff] ^ (crc >> 8);
             crc32c_table_little[k][n] = crc;
         }
@@ -345,8 +340,7 @@ static bool crc32c_once_big;
 static uint32_t crc32c_table_big_byte[256];
 static uint64_t crc32c_table_big[8][256];
 static void crc32c_init_sw_big(void) {
-    unsigned n;
-    for (n = 0; n < 256; n++) {
+    for (unsigned n = 0; n < 256; n++) {
         uint32_t crc = n;
         crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
         crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
@@ -358,11 +352,10 @@ static void crc32c_init_sw_big(void) {
         crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
         crc32c_table_big_byte[n] = crc;
     }
-    for (n = 0; n < 256; n++) {
+    for (unsigned n = 0; n < 256; n++) {
         uint32_t crc = crc32c_table_big_byte[n];
         crc32c_table_big[0][n] = swap(crc);
-        unsigned k;
-        for (k = 1; k < 8; k++) {
+        for (unsigned k = 1; k < 8; k++) {
             crc = crc32c_table_big_byte[crc & 0xff] ^ (crc >> 8);
             crc32c_table_big[k][n] = swap(crc);
         }

ccan/ccan/crypto/hmac_sha256/hmac_sha256.c

@@ -35,8 +35,7 @@ void hmac_sha256_init(struct hmac_sha256_ctx *ctx,
 	 *  (e.g., if K is of length 20 bytes and B=64, then K will be
 	 *   appended with 44 zero bytes 0x00)
 	 */
-	if (ksize != 0)
-		memcpy(k_ipad, k, ksize);
+	memcpy(k_ipad, k, ksize);
 	memset((char *)k_ipad + ksize, 0, HMAC_SHA256_BLOCKSIZE - ksize);
 
 	/*

ccan/ccan/fdpass/fdpass.c

@@ -3,7 +3,6 @@
 #include <sys/socket.h>
 #include <errno.h>
 #include <string.h>
-#include <sys/types.h>
 
 bool fdpass_send(int sockout, int fd)
 {

ccan/ccan/graphql/LICENSE

@@ -1 +0,0 @@
-../../licenses/BSD-MIT