Compare commits

...

2 Commits

Author SHA1 Message Date
nicolas.dorier
87f326b19e
Fix dockerfile build
Some checks failed
CI Compilation testing / test (alpine) (push) Has been cancelled
2022-10-10 13:27:02 +02:00
nicolas.dorier
6495c146bb
Adapt Dockerfile for BTCPay deployment 2022-10-10 13:23:48 +02:00
7 changed files with 310 additions and 19 deletions

144
.circleci/config.yml Normal file
View File

@ -0,0 +1,144 @@
version: 2
jobs:
# publish jobs require $DOCKERHUB_REPO, $DOCKERHUB_USER, $DOCKERHUB_PASS defined
amd64:
machine:
enabled: true
steps:
- checkout
- run:
command: |
LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
#
sudo docker build --pull -t $DOCKERHUB_REPO:$LATEST_TAG-amd64 -f Dockerfile .
sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-amd64
arm32v7:
machine:
enabled: true
steps:
- checkout
- run:
command: |
LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
# Make sure the builder is copy the arm emulator
sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset
sed -i -e 's/#EnableQEMU //g' "contrib/linuxarm32v7.Dockerfile"
#
sudo docker build --pull -t $DOCKERHUB_REPO:$LATEST_TAG-arm32v7 -f contrib/linuxarm32v7.Dockerfile .
sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-arm32v7
arm64v8:
machine:
enabled: true
steps:
- checkout
- run:
command: |
LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
# Make sure the builder is copy the arm emulator
sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset
sed -i -e 's/#EnableQEMU //g' "contrib/linuxarm64v8.Dockerfile"
#
sudo docker build --pull -t $DOCKERHUB_REPO:$LATEST_TAG-arm64v8 -f contrib/linuxarm64v8.Dockerfile .
sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-arm64v8
publish_docker_dev:
machine:
enabled: true
steps:
- checkout
- run:
command: |
LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
#
sudo docker build --build-arg "DEVELOPER=1" --build-arg "TRACE_TOOLS=false" -t "$DOCKERHUB_REPO:$LATEST_TAG-dev" .
sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-dev
publish_docker_bench:
machine:
enabled: true
steps:
- checkout
- run:
command: |
LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
#
sudo docker build --build-arg "DEVELOPER=1" --build-arg "TRACE_TOOLS=true" -t "$DOCKERHUB_REPO:$LATEST_TAG-bench" .
sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-bench
multiarch:
machine:
enabled: true
image: circleci/classic:201808-01
steps:
- run:
command: |
# Turn on Experimental features
sudo mkdir $HOME/.docker
sudo sh -c 'echo "{ \"experimental\": \"enabled\" }" >> $HOME/.docker/config.json'
#
sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
#
LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
sudo docker manifest create --amend $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-amd64 $DOCKERHUB_REPO:$LATEST_TAG-arm32v7 $DOCKERHUB_REPO:$LATEST_TAG-arm64v8
sudo docker manifest annotate $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-amd64 --os linux --arch amd64
sudo docker manifest annotate $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-arm32v7 --os linux --arch arm --variant v7
sudo docker manifest annotate $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-arm64v8 --os linux --arch arm64 --variant v8
sudo docker manifest push $DOCKERHUB_REPO:$LATEST_TAG -p
workflows:
version: 2
publish:
jobs:
- amd64:
filters:
# ignore any commit on any branch by default
branches:
ignore: /.*/
# only act on version tags
tags:
only: /basedon-.+/
- arm32v7:
filters:
# ignore any commit on any branch by default
branches:
ignore: /.*/
# only act on version tags
tags:
only: /basedon-.+/
- arm64v8:
filters:
# ignore any commit on any branch by default
branches:
ignore: /.*/
# only act on version tags
tags:
only: /basedon-.+/
- publish_docker_dev:
filters:
# ignore any commit on any branch by default
branches:
ignore: /.*/
# only act on version tags
tags:
only: /basedon-.+/
- publish_docker_bench:
filters:
# ignore any commit on any branch by default
branches:
ignore: /.*/
# only act on version tags
tags:
only: /basedon-.+/
- multiarch:
requires:
- amd64
- arm32v7
- arm64v8
filters:
branches:
ignore: /.*/
tags:
only: /basedon-.+/

3
.dockerignore Normal file
View File

@ -0,0 +1,3 @@
Dockerfile
linuxarm32v7.Dockerfile
.circleci/config.yml

View File

@ -45,6 +45,13 @@ RUN mkdir /opt/litecoin && cd /opt/litecoin \
&& tar -xzvf litecoin.tar.gz $BD/litecoin-cli --strip-components=1 --exclude=*-qt \
&& rm litecoin.tar.gz
ENV DESCHASHPLUGIN_URL https://github.com/fiatjaf/sparko/releases/download/invoicewithdescriptionhash-v1.2/invoicewithdescriptionhash_linux_amd64
ENV DESCHASHPLUGIN_SHA256 E3EA0D076A26D774BA68D1D5E3FE48D267CE02D077933EF3CBAE1FC39007FB11
RUN mkdir /opt/deschashplugin && cd /opt/deschashplugin \
&& wget -qO invoicewithdescriptionhash "$DESCHASHPLUGIN_URL" \
&& echo "$DESCHASHPLUGIN_SHA256 invoicewithdescriptionhash" | sha256sum -c - \
&& chmod a+x invoicewithdescriptionhash
FROM debian:bullseye-slim as builder
ENV LIGHTNINGD_VERSION=master
@ -108,13 +115,27 @@ RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/inst
&& pip3 install -U wheel \
&& /root/.local/bin/poetry config virtualenvs.create false \
&& /root/.local/bin/poetry install
RUN pip3 install mrkd
RUN ./configure --prefix=/tmp/lightning_install --enable-static && make -j3 DEVELOPER=${DEVELOPER} && make install
FROM debian:bullseye-slim as final
ARG TRACE_TOOLS=false
ENV TRACE_TOOLS=$TRACE_TOOLS
ENV TRACE_LOCATION=/opt/traces
VOLUME /opt/traces
COPY --from=downloader /opt/tini /usr/bin/tini
RUN apt-get update && apt-get install -y --no-install-recommends socat inotify-tools python3 python3-pip libpq5\
&& \
( ! $TRACE_TOOLS || \
( \
apt-get install -y --no-install-recommends perl linux-base curl ca-certificates && \
mkdir FlameGraph && cd FlameGraph && \
curl -Lo FlameGraph.tar.gz "https://github.com/brendangregg/FlameGraph/archive/v1.0.tar.gz" && \
tar -zxvf FlameGraph.tar.gz --strip-components=1 && rm FlameGraph.tar.gz && cd .. \
) \
) \
&& rm -rf /var/lib/apt/lists/*
ENV LIGHTNINGD_DATA=/root/.lightning
@ -123,11 +144,15 @@ ENV LIGHTNINGD_PORT=9735
ENV LIGHTNINGD_NETWORK=bitcoin
RUN mkdir $LIGHTNINGD_DATA && \
mkdir /etc/bundledplugins && \
mkdir $LIGHTNINGD_DATA/plugins && \
touch $LIGHTNINGD_DATA/config
VOLUME [ "/root/.lightning" ]
COPY --from=builder /tmp/lightning_install/ /usr/local/
COPY --from=downloader /opt/bitcoin/bin /usr/bin
COPY --from=downloader /opt/litecoin/bin /usr/bin
COPY --from=downloader /opt/deschashplugin $LIGHTNINGD_DATA/plugins
COPY --from=downloader /opt/deschashplugin /etc/bundledplugins
COPY tools/docker-entrypoint.sh entrypoint.sh
EXPOSE 9735 9835

View File

@ -1,5 +1,5 @@
# This dockerfile is meant to cross compile with a x64 machine for a arm32v7 host
# It is using multi stage build:
# It is using multi stage build:
# * downloader: Download litecoin/bitcoin and qemu binaries needed for core-lightning
# * builder: Cross compile c-lightning dependencies, then c-lightning itself with static linking
# * final: Copy the binaries required at runtime
@ -45,6 +45,13 @@ RUN mkdir /opt/litecoin && cd /opt/litecoin \
&& tar -xzvf litecoin.tar.gz $BD/litecoin-cli --strip-components=1 --exclude=*-qt \
&& rm litecoin.tar.gz
ENV DESCHASHPLUGIN_URL https://github.com/fiatjaf/sparko/releases/download/invoicewithdescriptionhash-v1.2/invoicewithdescriptionhash_linux_arm
ENV DESCHASHPLUGIN_SHA256 D268BFAD4CCDA9670A17ED9FACC79F81CDD22756C76ED4A94BE59282E4E819F0
RUN mkdir /opt/deschashplugin && cd /opt/deschashplugin \
&& wget -qO invoicewithdescriptionhash "$DESCHASHPLUGIN_URL" \
&& echo "$DESCHASHPLUGIN_SHA256 invoicewithdescriptionhash" | sha256sum -c - \
&& chmod a+x invoicewithdescriptionhash
FROM debian:buster-slim as builder
ENV LIGHTNINGD_VERSION=master
@ -83,7 +90,6 @@ RUN wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz \
&& ./configure --disable-assembly --prefix=$QEMU_LD_PREFIX --host=${target_host} \
&& make \
&& make install && cd .. && rm gmp-6.1.2.tar.xz && rm -rf gmp-6.1.2
COPY --from=downloader /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static
WORKDIR /opt/lightningd
COPY . /tmp/lightning
@ -92,13 +98,14 @@ RUN git clone --recursive /tmp/lightning . && \
ARG DEVELOPER=0
ENV PYTHON_VERSION=3
RUN pip3 install mrkd
RUN ./configure --prefix=/tmp/lightning_install --enable-static && make -j3 DEVELOPER=${DEVELOPER} && make install
FROM arm32v7/debian:buster-slim as final
COPY --from=downloader /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static
COPY --from=downloader /opt/tini /usr/bin/tini
RUN apt-get update && apt-get install -y --no-install-recommends socat inotify-tools python3 python3-pip \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/*
ENV LIGHTNINGD_DATA=/root/.lightning
ENV LIGHTNINGD_RPC_PORT=9835
@ -106,11 +113,15 @@ ENV LIGHTNINGD_PORT=9735
ENV LIGHTNINGD_NETWORK=bitcoin
RUN mkdir $LIGHTNINGD_DATA && \
mkdir /etc/bundledplugins && \
mkdir $LIGHTNINGD_DATA/plugins && \
touch $LIGHTNINGD_DATA/config
VOLUME [ "/root/.lightning" ]
COPY --from=builder /tmp/lightning_install/ /usr/local/
COPY --from=downloader /opt/bitcoin/bin /usr/bin
COPY --from=downloader /opt/litecoin/bin /usr/bin
COPY --from=downloader /opt/deschashplugin $LIGHTNINGD_DATA/plugins
COPY --from=downloader /opt/deschashplugin /etc/bundledplugins
COPY tools/docker-entrypoint.sh entrypoint.sh
EXPOSE 9735 9835

View File

@ -1,5 +1,5 @@
# This dockerfile is meant to cross compile with a x64 machine for a arm64v8 host
# It is using multi stage build:
# It is using multi stage build:
# * downloader: Download litecoin/bitcoin and qemu binaries needed for Core Lightning
# * builder: Cross compile Core Lightning dependencies, then Core Lightning itself with static linking
# * final: Copy the binaries required at runtime
@ -45,6 +45,13 @@ RUN mkdir /opt/litecoin && cd /opt/litecoin \
&& tar -xzvf litecoin.tar.gz $BD/litecoin-cli --strip-components=1 --exclude=*-qt \
&& rm litecoin.tar.gz
ENV DESCHASHPLUGIN_URL https://github.com/fiatjaf/sparko/releases/download/invoicewithdescriptionhash-v1.2/invoicewithdescriptionhash_linux_arm64
ENV DESCHASHPLUGIN_SHA256 3953CD545D7B40CBCDF3C1CBF4AA779832290A95DDBF319E992C2692BC1C3F39
RUN mkdir /opt/deschashplugin && cd /opt/deschashplugin \
&& wget -qO invoicewithdescriptionhash "$DESCHASHPLUGIN_URL" \
&& echo "$DESCHASHPLUGIN_SHA256 invoicewithdescriptionhash" | sha256sum -c - \
&& chmod a+x invoicewithdescriptionhash
FROM debian:buster-slim as builder
ENV LIGHTNINGD_VERSION=master
@ -91,13 +98,14 @@ RUN git clone --recursive /tmp/lightning . && \
ARG DEVELOPER=0
ENV PYTHON_VERSION=3
RUN pip3 install mrkd
RUN ./configure --prefix=/tmp/lightning_install --enable-static && make -j3 DEVELOPER=${DEVELOPER} && make install
FROM arm64v8/debian:buster-slim as final
COPY --from=downloader /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static
COPY --from=downloader /opt/tini /usr/bin/tini
RUN apt-get update && apt-get install -y --no-install-recommends socat inotify-tools python3 python3-pip \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/*
ENV LIGHTNINGD_DATA=/root/.lightning
ENV LIGHTNINGD_RPC_PORT=9835
@ -105,11 +113,15 @@ ENV LIGHTNINGD_PORT=9735
ENV LIGHTNINGD_NETWORK=bitcoin
RUN mkdir $LIGHTNINGD_DATA && \
mkdir /etc/bundledplugins && \
mkdir $LIGHTNINGD_DATA/plugins && \
touch $LIGHTNINGD_DATA/config
VOLUME [ "/root/.lightning" ]
COPY --from=builder /tmp/lightning_install/ /usr/local/
COPY --from=downloader /opt/bitcoin/bin /usr/bin
COPY --from=downloader /opt/litecoin/bin /usr/bin
COPY --from=downloader /opt/deschashplugin $LIGHTNINGD_DATA/plugins
COPY --from=downloader /opt/deschashplugin /etc/bundledplugins
COPY tools/docker-entrypoint.sh entrypoint.sh
EXPOSE 9735 9835

View File

@ -865,6 +865,7 @@ static void wait_and_check_bitcoind(struct plugin *p)
pid_t child;
const char **cmd = gather_args(bitcoind, "getnetworkinfo", NULL);
bool printed = false;
bool isWarmup = false;
char *output = NULL;
for (;;) {
@ -900,17 +901,20 @@ static void wait_and_check_bitcoind(struct plugin *p)
/* bitcoin/src/rpc/protocol.h:
* RPC_IN_WARMUP = -28, //!< Client still warming up
*/
if (WEXITSTATUS(status) != 28) {
if (WEXITSTATUS(status) == 1)
bitcoind_failure(p, "Could not connect to bitcoind using"
" bitcoin-cli. Is bitcoind running?");
bitcoind_failure(p, tal_fmt(bitcoind, "%s exited with code %i: %s",
cmd[0], WEXITSTATUS(status), output));
}
isWarmup = WEXITSTATUS(status) == 28;
if (!printed) {
plugin_log(p, LOG_UNUSUAL,
"Waiting for bitcoind to warm up...");
if (isWarmup)
{
plugin_log(p, LOG_UNUSUAL,
"Waiting for bitcoind to warm up...");
}
else
{
plugin_log(p, LOG_UNUSUAL,
tal_fmt(bitcoind, "%s exited with code %i: %s... retrying",
cmd[0], WEXITSTATUS(status), output));
}
printed = true;
}
sleep(1);

View File

@ -2,20 +2,112 @@
: "${EXPOSE_TCP:=false}"
networkdatadir="${LIGHTNINGD_DATA}/${LIGHTNINGD_NETWORK}"
cat <<-EOF > "$LIGHTNINGD_DATA/config"
${LIGHTNINGD_OPT}
bind-addr=0.0.0.0:${LIGHTNINGD_PORT}
EOF
: "${EXPOSE_TCP:=false}"
LIGHTNINGD_NETWORK_NAME=""
if [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "mainnet" ]; then
LIGHTNINGD_NETWORK_NAME="bitcoin"
elif [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "testnet" ]; then
LIGHTNINGD_NETWORK_NAME="testnet"
elif [ "$LIGHTNINGD_CHAIN" == "btc" ] && [ "$LIGHTNINGD_NETWORK" == "regtest" ]; then
LIGHTNINGD_NETWORK_NAME="regtest"
elif [ "$LIGHTNINGD_CHAIN" == "ltc" ] && [ "$LIGHTNINGD_NETWORK" == "mainnet" ]; then
LIGHTNINGD_NETWORK_NAME="litecoin"
elif [ "$LIGHTNINGD_CHAIN" == "ltc" ] && [ "$LIGHTNINGD_NETWORK" == "testnet" ]; then
LIGHTNINGD_NETWORK_NAME="litecoin-testnet"
else
echo "Invalid combinaion of LIGHTNINGD_NETWORK and LIGHTNINGD_CHAIN. LIGHTNINGD_CHAIN should be btc or ltc. LIGHTNINGD_NETWORK should be mainnet, testnet or regtest."
echo "ltc regtest is not supported"
exit
fi
echo "network=$LIGHTNINGD_NETWORK_NAME" >> "$LIGHTNINGD_DATA/config"
echo "network=$LIGHTNINGD_NETWORK_NAME added in $LIGHTNINGD_DATA/config"
if [[ $TRACE_TOOLS == "true" ]]; then
echo "Trace tools detected, installing sample.sh..."
echo 0 > /proc/sys/kernel/kptr_restrict
echo "
# This script will take one minute of stacktrace samples and plot it in a flamegraph
LIGHTNING_PROCESSES=\$(pidof lightningd lightning_chann lightning_closi lightning_gossi lightning_hsmd lightning_oncha lightning_openi lightning_hsmd lightning_gossipd lightning_channeld | sed -e 's/\s/,/g')
perf record -F 99 -g -a --pid \$LIGHTNING_PROCESSES -o \"$TRACE_LOCATION/perf.data\" -- sleep 60
perf script -i \"$TRACE_LOCATION/perf.data\" > \"$TRACE_LOCATION/output.trace\"
cd /FlameGraph
./stackcollapse-perf.pl \"$TRACE_LOCATION/output.trace\" > \"$TRACE_LOCATION/output.trace.folded\"
svg=\"$TRACE_LOCATION/\$((\$SECONDS / 60))min.svg\"
./flamegraph.pl \"$TRACE_LOCATION/output.trace.folded\" > \"\$svg\"
rm \"$TRACE_LOCATION/perf.data\"
rm \"$TRACE_LOCATION/output.trace\"
rm \"$TRACE_LOCATION/output.trace.folded\"
echo \"flamegraph taken: \$svg\"
" > /usr/bin/sample.sh
chmod +x /usr/bin/sample.sh
echo "
# This script will run sample.sh after 2 min then every 10 minutes
sleep 120
sample.sh
while true; do
sleep 300
. sample.sh
done
" > /usr/bin/sample-loop.sh
chmod +x /usr/bin/sample-loop.sh
fi
if [[ "${LIGHTNINGD_ANNOUNCEADDR}" ]]; then
echo "announce-addr=$LIGHTNINGD_ANNOUNCEADDR:${LIGHTNINGD_PORT}" >> "$LIGHTNINGD_DATA/config"
fi
if [[ "${LIGHTNINGD_ALIAS}" ]]; then
# This allow to strip this parameter if LND_ALIGHTNINGD_ALIASLIAS is empty or null, and truncate it
LIGHTNINGD_ALIAS="$(echo "$LIGHTNINGD_ALIAS" | cut -c -32)"
echo "alias=$LIGHTNINGD_ALIAS" >> "$LIGHTNINGD_DATA/config"
echo "alias=$LIGHTNINGD_ALIAS added to $LIGHTNINGD_DATA/config"
fi
if [[ "${LIGHTNINGD_READY_FILE}" ]]; then
echo "Waiting $LIGHTNINGD_READY_FILE to be created..."
while [ ! -f "$LIGHTNINGD_READY_FILE" ]; do sleep 1; done
echo "The chain is fully synched"
fi
if [[ "${LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE}" ]]; then
echo "Waiting $LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE to be created by tor..."
while [ ! -f "$LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE" ]; do sleep 1; done
HIDDENSERVICE_ONION="$(head -n 1 "$LIGHTNINGD_HIDDENSERVICE_HOSTNAME_FILE"):${LIGHTNINGD_PORT}"
echo "announce-addr=$HIDDENSERVICE_ONION" >> "$LIGHTNINGD_DATA/config"
echo "announce-addr=$HIDDENSERVICE_ONION added to $LIGHTNINGD_DATA/config"
fi
if ! grep -q "^rpc-file=" "$LIGHTNINGD_DATA/config"; then
echo "rpc-file=$LIGHTNINGD_DATA/lightning-rpc" >> "$LIGHTNINGD_DATA/config"
echo "rpc-file=$LIGHTNINGD_DATA/lightning-rpc added to $LIGHTNINGD_DATA/config"
fi
echo "Installing bundled plugins"
mkdir -p "$LIGHTNINGD_DATA/plugins"
cp -u /etc/bundledplugins/* $LIGHTNINGD_DATA/plugins/
echo "C-Lightning starting, listening on port ${LIGHTNINGD_PORT}"
if [ "$EXPOSE_TCP" == "true" ]; then
set -m
lightningd "$@" &
echo "Core-Lightning starting"
while read -r i; do if [ "$i" = "lightning-rpc" ]; then break; fi; done \
< <(inotifywait -e create,open --format '%f' --quiet "${networkdatadir}" --monitor)
< <(inotifywait -e create,open --format '%f' --quiet "$LIGHTNINGD_DATA" --monitor)
echo "Core-Lightning started"
echo "Core-Lightning started, RPC available on port $LIGHTNINGD_RPC_PORT"
socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:${networkdatadir}/lightning-rpc" &
socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:$LIGHTNINGD_DATA/lightning-rpc" &
fg %-
else
exec lightningd --network="${LIGHTNINGD_NETWORK}" "$@"
exec lightningd "$@"
fi