V2.5.0 update, and replace submodule with docker images (#21)

* bump to v2.5.0

* wip

* wip

* v2.5.0 with docker images

* removed submodule

* fix CLN

* v2.5.0

* update deno deps

---------

Co-authored-by: islandbitcoin <dread@start9.com>

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "mempool"]
-	path = mempool
-	url = https://github.com/mempool/mempool.git

Dockerfile

@@ -1,66 +1,55 @@
-FROM node:16-buster-slim AS builder
+# Use a multi-stage build to combine the specific images
+FROM mempool/frontend:latest AS frontend
+FROM mempool/backend:latest AS backend
+FROM mariadb:10.5.8 AS db
+FROM node:16.16.0-buster-slim AS runner
 
+ENV MEMPOOL_CLEAR_PROTECTION_MINUTES "20"
+ENV MEMPOOL_INDEXING_BLOCKS_AMOUNT "52560"
+ENV MEMPOOL_STDOUT_LOG_MIN_PRIORITY "info"
+ENV LIGHTNING_STATS_REFRESH_INTERVAL 3600
+ENV LIGHTNING_GRAPH_REFRESH_INTERVAL 3600
+
+USER root
 # arm64 or amd64
 ARG PLATFORM
 # aarch64 or x86_64
 ARG ARCH
+# Install necessary packages
+RUN apt-get update && \
+    apt-get install -y nginx wait-for-it wget curl netcat \
+    build-essential python3 pkg-config rsync gettext \
+    mariadb-server mariadb-client libaio1 iproute2 pwgen \
+    && wget https://github.com/mikefarah/yq/releases/download/v4.6.3/yq_linux_${PLATFORM}.tar.gz -O - |\
+      tar xz && mv yq_linux_${PLATFORM} /usr/bin/yq \
+    && apt-get clean
+# RUN groupadd -r mysql && useradd -r -g mysql mysql
 
-WORKDIR /build
-COPY mempool/ .
-# because just a submodule in wrapper project
-RUN rm .git && sh docker/init.sh
-# generate-config.js uses this ref
-COPY .git/modules/mempool/refs/heads/master .git/refs/heads/master
+WORKDIR /patch
 
-RUN apt-get update && apt-get install -y build-essential python3 pkg-config rsync
-
-WORKDIR /build/frontend
-RUN npm i && npm run build
-
-WORKDIR /build/backend
-RUN npm ci --production && npm i typescript && npm run build
-
-WORKDIR /build
-RUN cp docker/backend/mempool-config.json backend/
-
-FROM node:16-buster-slim
-
-# arm64 or amd64
-ARG PLATFORM
-# aarch64 or x86_64
-ARG ARCH
+# Copy frontend files
+COPY --from=frontend /patch/entrypoint.sh .
+COPY --from=frontend /patch/wait-for .
+COPY --from=frontend /var/www/mempool /var/www/mempool
+COPY --from=frontend /etc/nginx/nginx.conf /etc/nginx/
+COPY --from=frontend /etc/nginx/conf.d/nginx-mempool.conf /etc/nginx/conf.d/
 
 WORKDIR /backend
 
-RUN apt-get update && apt-get install wget netcat jq pwgen vim procps nginx curl mariadb-server mariadb-client -y
-RUN wget https://github.com/mikefarah/yq/releases/download/v4.6.3/yq_linux_${PLATFORM}.tar.gz -O - |\
-  tar xz && mv yq_linux_${PLATFORM} /usr/bin/yq
+# Copy backend files
+COPY --from=backend /backend/package ./package/
+COPY --from=backend /backend/GeoIP ./GeoIP/
+COPY --from=backend /backend/mempool-config.json /backend/start.sh /backend/wait-for-it.sh ./
 
-USER root
+# Create data folder for cache and MySQL data
+RUN mkdir -p /build/data/cache /var/lib/mysql/data
 
-COPY --from=builder /build/backend .
-COPY --from=builder /build/frontend/wait-for /usr/local/bin/wait-for.sh
-COPY --from=builder /build/frontend/dist/mempool /var/www/mempool
-COPY --from=builder /build/nginx.conf /etc/nginx/
-COPY --from=builder /build/nginx-mempool.conf /etc/nginx/conf.d/
-COPY --from=builder /build/nginx-mempool.conf /etc/nginx/nginx-mempool.conf
-RUN cp wait-for-it.sh /usr/local/bin && chmod +x start.sh && chmod +x /backend/wait-for-it.sh && chmod +x /usr/local/bin/wait-for.sh \
-        # initalize db folders
-        && mkdir -p data mysql/data mysql/db-scripts \
-        && mkdir /var/cache/nginx \
-        && touch /var/run/nginx.pid \
-        && chown -R 1000:1000 /backend && chmod -R 755 /backend && \
-        chown -R 1000:1000 /var/cache/nginx && \
-        chown -R 1000:1000 /var/log/nginx && \
-        chown -R 1000:1000 /etc/nginx/nginx.conf && \
-        chown -R 1000:1000 /etc/nginx/conf.d \
-        && touch /var/run/nginx.pid && \
-        chown -R 1000:1000 /var/run/nginx.pid
+# Set user and group for the folders
+RUN chown -R mysql:mysql /build/data /var/lib/mysql/data
 
 # BUILD S9 CUSTOM
 ADD ./docker_entrypoint.sh /usr/local/bin/docker_entrypoint.sh
-ADD assets/utils/health-check.sh /usr/local/bin/health-check.sh
-ADD assets/utils/check-synced.sh /usr/local/bin/check-synced.sh
+ADD assets/utils/*.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/*.sh
 
 # remove to we can manually handle db initalization

Makefile

@@ -15,7 +15,6 @@ install: $(PKG_ID).s9pk
 
 clean:
 	rm -rf docker-images
-	rm -f image.tar
 	rm -f $(PKG_ID).s9pk
 	rm -f scripts/*.js
 
@@ -31,5 +30,4 @@ docker-images/aarch64.tar: Dockerfile docker_entrypoint.sh assets/utils/*
 	docker buildx build --tag start9/$(PKG_ID)/main:$(PKG_VERSION) --platform=linux/arm64 --build-arg PLATFORM=arm64 -o type=docker,dest=docker-images/aarch64.tar .
 
 $(PKG_ID).s9pk: manifest.yaml instructions.md LICENSE icon.png scripts/embassy.js docker-images/aarch64.tar docker-images/x86_64.tar
-	if ! [ -z "$(ARCH)" ]; then cp docker-images/$(ARCH).tar image.tar; fi
 	embassy-sdk pack

docker_entrypoint.sh

@@ -10,6 +10,7 @@ _term() {
 
 # FRONTEND SETUP
 
+LIGHTNING_DETECTED_PORT=9735
 __MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST:=127.0.0.1}
 __MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999}
 __MEMPOOL_FRONTEND_HTTP_PORT__=${FRONTEND_HTTP_PORT:=8080}
@@ -22,6 +23,7 @@ cp /etc/nginx/conf.d/nginx-mempool.conf /etc/nginx/nginx-mempool.conf
 cp /etc/nginx/nginx.conf /backend/nginx.conf
 sed -i -e "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" -e "s/127.0.0.1://" -e "/listen/a\                server_name 127.0.0.1;" -e "s/listen 80/listen 8080/g" /backend/nginx.conf
 cat /backend/nginx.conf > /etc/nginx/nginx.conf
+/patch/entrypoint.sh
 
 #  BACKEND SETUP
 
@@ -41,6 +43,29 @@ fi
 sed -i "s/CORE_RPC_HOST:=127.0.0.1/CORE_RPC_HOST:=$bitcoind_host/" start.sh
 sed -i "s/CORE_RPC_USERNAME:=mempool/CORE_RPC_USERNAME:=$bitcoind_user/" start.sh
 sed -i "s/CORE_RPC_PASSWORD:=mempool/CORE_RPC_PASSWORD:=$bitcoind_pass/" start.sh
+
+# Configure mempool to set lightning to true if lightning is enabled
+if [ "$(yq e ".lightning.type" /root/start9/config.yaml)" = "none" ]; then
+	sed -i 's/LIGHTNING_ENABLED:=true/LIGHTNING_ENABLED:=false/' start.sh
+	echo "Lightning tab disabled..."
+else
+	sed -i 's/LIGHTNING_ENABLED:=false/LIGHTNING_ENABLED:=true/' start.sh
+	echo "Lightning tab enabled..."
+fi
+
+# Allow user to choose between LND and CLN nodes for lightning
+if [ "$(yq e ".lightning.type" /root/start9/config.yaml)" = "lnd" ]; then
+	sed -i 's/LIGHTNING_BACKEND:=\"cln\"/LIGHTNING_BACKEND:=\"lnd\"/' start.sh
+	sed -i 's/LND_TLS_CERT_PATH:=\"\"/LND_TLS_CERT_PATH:=\"\/mnt\/lnd\/tls.cert\"/' start.sh
+	sed -i 's/LND_MACAROON_PATH:=\"\"/LND_MACAROON_PATH:=\"\/mnt\/lnd\/readonly.macaroon\"/' start.sh
+	sed -i 's/LND_REST_API_URL:=\"https:\/\/localhost:8080\"/LND_REST_API_URL:=\"https:\/\/lnd.embassy:8080\"/' start.sh
+	echo "Running on LND..."
+elif [ "$(yq e ".lightning.type" /root/start9/config.yaml)" = "cln" ]; then
+	sed -i 's/LIGHTNING_BACKEND:=\"lnd\"/LIGHTNING_BACKEND:=\"cln\"/' start.sh
+	sed -i 's/CLIGHTNING_SOCKET:=\"\"/CLIGHTNING_SOCKET:=\"\/mnt\/c-lightning\/lightning-rpc\"/' start.sh
+	echo "Running on Core Lightning..."
+fi
+
 if [ "$(yq e ".enable-electrs" /root/start9/config.yaml)" = "true" ]; then
 	sed -i 's/ELECTRUM_HOST:=127.0.0.1/ELECTRUM_HOST:=electrs.embassy/' start.sh
 	sed -i 's/ELECTRUM_PORT:=50002/ELECTRUM_PORT:=50001/' start.sh
@@ -49,8 +74,8 @@ else
 	sed -i '/^node \/backend\/dist\/index.js/i jq \x27.MEMPOOL.BACKEND="none"\x27 \/backend\/mempool-config.json > \/backend\/mempool-config.json.tmp && mv \/backend\/mempool-config.json.tmp \/backend\/mempool-config.json' start.sh
 	sed -i 's/MEMPOOL_BACKEND:=electrum/MEMPOOL_BACKEND:=none/' start.sh
 fi
-# DATABASE SETUP
 
+# DATABASE SETUP
 if [ -d "/run/mysqld" ]; then
 	echo "[i] mysqld already present, skipping creation"
 	chown -R mysql:mysql /run/mysqld
@@ -136,7 +161,7 @@ sed -i "s/user nobody;//g" /etc/nginx/nginx.conf
 nginx -g 'daemon off;' &
 frontend_process=$!
 
-/backend/wait-for-it.sh 127.0.0.1:3306 --timeout=60 --strict -- /backend/start.sh &
+/backend/wait-for-it.sh localhost:3306 --timeout=720 --strict -- ./start.sh &
 backend_process=$!
 
 echo 'All processes initalized'

manifest.yaml

@@ -1,8 +1,9 @@
 id: mempool
 title: Mempool
-version: 2.4.0.3
+version: 2.5.0
 release-notes: | 
-  * Updated Mempool to run x86_64 architecture
+  * Update Mempool to v2.5.0 [Release Notes](https://github.com/mempool/mempool/releases/tag/v2.5.0)
+    * Note: After upgrading the blocks database table will be cleared and reindexed
 license: AGPL
 wrapper-repo: "https://github.com/Start9Labs/mempool-wrapper"
 upstream-repo: "https://github.com/mempool/mempool"
@@ -24,6 +25,10 @@ main:
   args: []
   mounts:
     main: /root
+    cache: /backend/cache
+    db: /var/lib/mysql
+    lnd: /mnt/lnd
+    c-lightning: "/mnt/c-lightning"
 health-checks:
   web-ui:
     name: Web Interface
@@ -87,9 +92,39 @@ dependencies:
       type: "opt-out"
       how: Set Enable Electrs to Disabled in the config
     description: Used to provide an index for address lookups
+  lnd:
+    version: ">=0.14.3 <0.17.0"
+    description: Used to communicate with the Lightning Network.
+    requirement:
+      type: "opt-in"
+      how: Use the LND instance by default
+    config: ~
+  c-lightning:
+    version: ">=0.10.1 <24.0.0"
+    description: Used to communicate with the Lightning Network.
+    requirement:
+      type: "opt-in"
+      how: Can opt to use the CLN instance instead of LND
+    config: ~
 volumes: 
   main:
-    type: data 
+    type: data
+  cache:
+    type: data
+  db:
+    type: data
+  lnd:
+    type: pointer
+    package-id: lnd
+    volume-id: main
+    path: "/public"
+    readonly: true
+  c-lightning:
+    type: pointer
+    package-id: c-lightning
+    volume-id: main
+    path: /shared
+    readonly: true
 alerts:
   start: |
     READ CAREFULLY! When first running Mempool, previous block fee estimates will show as zero values until the service is able to catch up. This is expected behaviour.
@@ -123,6 +158,8 @@ backup:
     mounts:
       BACKUP: /mnt/backup
       main: /root/data
+      cache: /backend/cache
+      db: /var/lib/mysql
     io-format: yaml
   restore:
     type: docker
@@ -137,6 +174,8 @@ backup:
     mounts:
       BACKUP: /mnt/backup
       main: /root/data
+      cache: /backend/cache
+      db: /var/lib/mysql
     io-format: yaml
 migrations:
   from:

mempool

@@ -1 +0,0 @@
-Subproject commit f901f06992195c577cc829f917067393e5893edc

scripts/deps.ts

@@ -1,4 +1,4 @@
-export * from "https://deno.land/x/embassyd_sdk@v0.3.3.0.4/mod.ts";
-export * from "https://deno.land/x/embassyd_sdk@v0.3.3.0.4/util.ts";
-export * from "https://deno.land/x/embassyd_sdk@v0.3.3.0.4/healthUtil.ts";
-export * from "https://deno.land/x/embassyd_sdk@v0.3.1.0.12/emvar-lite/mod.ts";
+export * from "https://deno.land/x/embassyd_sdk@v0.3.3.0.5/mod.ts";
+export * from "https://deno.land/x/embassyd_sdk@v0.3.3.0.5/util.ts";
+export * from "https://deno.land/x/embassyd_sdk@v0.3.3.0.5/healthUtil.ts";
+export * from "https://deno.land/x/embassyd_sdk@v0.3.1.0.12/emvar-lite/mod.ts";

scripts/embassy.ts

@@ -4,4 +4,4 @@ export { getConfig } from "./services/getConfig.ts";
 export { dependencies } from "./services/dependencies.ts";
 export { migration } from "./services/migrations.ts";
 export { health } from "./services/healthChecks.ts";
-// export { main } from "./services/main.ts";
+// export { main } from "./services/main.ts";

scripts/migrations/2_5_0_down_migration.ts

@@ -0,0 +1,32 @@
+import { types as T, matches, YAML } from "../deps.ts"
+
+const { shape, string } = matches
+
+// add a const for the new lightning type string in getConfig
+const matchLightningType = shape({
+    lightning: shape({
+        type: string?.optional()
+    })
+})
+
+export const migration_down_2_5_0: T.ExpectedExports.migration = async (effects, _version) => {
+    await effects.createDir({
+        volumeId: "main",
+        path: "start9"
+    })
+    const config = await effects.readFile({
+        volumeId: "main",
+        path: "start9/config.yaml"
+    })
+    const parsed = YAML.parse(config)
+
+    // delete lightning.type if exists in config
+    if (matchLightningType.test(parsed)) {
+        if (parsed.lightning) {
+            delete parsed.lightning["type"];
+        }
+    }
+
+
+    return { result: { configured: true } }
+}

scripts/migrations/2_5_0_up_migration.ts

@@ -0,0 +1,34 @@
+// add migration script for new options in getConfig
+import { types as T, YAML, matches } from "../deps.ts"
+
+const { shape, string } = matches
+
+// add a const for the new lightning type string in getConfig
+const matchLightningType = shape({
+    lightning: shape({
+        type: string
+    })
+})
+
+
+
+
+export const migration_up_2_5_0: T.ExpectedExports.migration = async (effects, _version) => {
+    await effects.createDir({
+      volumeId: "main",
+      path: "start9"
+    })
+    const config = await effects.readFile({
+      volumeId: "main",
+      path: "start9/config.yaml"
+    })
+    const parsed = YAML.parse(config)
+  
+    // add lightning.type if it doesn't exist and set it to "lnd" as default
+    if (!matchLightningType.test(parsed)) {
+        return { result: { configured: false } }
+    }
+  
+    return { result: { configured: false } }
+  
+  }

scripts/services/dependencies.ts

@@ -79,8 +79,10 @@ const proxyChecks: Array<Check> = [
     "getblockchaininfo",
     "getblockhash",
     "getblock",
+    "getblockheader",
     "getmempoolentry",
     "getrawtransaction",
+    "decoderawtransaction",
     "getrawmempool",
     "gettxout",
     "validateaddress",

scripts/services/getConfig.ts

@@ -25,7 +25,7 @@ export const getConfig: T.ExpectedExports.getConfig = compat.getConfig({
       "description":
         "The Bitcoin Core node to connect to",
     },
-    "default": "internal-proxy",
+    "default": "internal",
     "variants": {
       "internal": {
         "user": {
@@ -83,10 +83,33 @@ export const getConfig: T.ExpectedExports.getConfig = compat.getConfig({
       },
     },
   },
+  "lightning": {
+    "type": "union",
+    "name": "Lightning Node",
+    "description":
+      "The Lightning node to connect to",
+    "tag": {
+      "id": "type",
+      "name": "Select Lightning Node",
+      "variant-names": {
+        "none": "Disabled",
+        "lnd": "LND",
+        "cln": "Core Lightning",
+      },
+      "description":
+        "The Lightning node to connect to",
+    },
+    "default": "none",
+    "variants": {
+      "none": {},
+      "lnd": {},
+      "cln": {},
+    }
+  },
   "enable-electrs": {
     "name": "Enable Electrs Address Lookups",
     "description": "Enables address lookups via an internal electrs instance",
     "type": "boolean",
     "default": true,
-  },
-});
+  }
+});

scripts/services/migrations.ts

@@ -1,6 +1,8 @@
 import { types as T, rangeOf } from "../deps.ts"
 import { migration_down_2_3_1_4 } from "../migrations/2_3_1_4_down_migration.ts";
 import { migration_up_2_3_1_4 } from "../migrations/2_3_1_4_up_migration.ts";
+import { migration_down_2_5_0 } from "../migrations/2_5_0_down_migration.ts";
+import { migration_up_2_5_0 } from "../migrations/2_5_0_up_migration.ts";
 
 export const migration: T.ExpectedExports.migration = async (effects, version) => {
 
@@ -10,12 +12,23 @@ export const migration: T.ExpectedExports.migration = async (effects, version) =
     return result
   }
 
+  if (rangeOf('<2.5.0').check(version)) {
+    const result = await migration_up_2_5_0(effects, version)
+    return result
+  }
+
   // to migrations (downgrades)
   if (rangeOf('>2.3.1.4').check(version)) {
     const result = await migration_down_2_3_1_4(effects, version)
     return result
   }
 
+  if (rangeOf('>2.5.0').check(version)) {
+    const result = await migration_down_2_5_0(effects, version)
+    return result
+  }
+
+
   return { result: { configured: true } }
 
 }

scripts/services/setConfig.ts

@@ -1,18 +1,33 @@
 import { compat, types as T } from "../deps.ts";
 
+// Define a custom type for T.Config to include the 'lightning' property with a 'type' property
+interface CustomConfig extends T.Config {
+  lightning?: {
+    type?: string;
+  };
+}
 // deno-lint-ignore require-await
 export const setConfig: T.ExpectedExports.setConfig = async (
   effects: T.Effects,
-  newConfig: T.Config,
+  newConfig: CustomConfig
 ) => {
-  const dependsOnElectrs: { [key: string]: string[] } =
-    newConfig?.["enable-electrs"] ? { electrs: ["synced"] } : {};
+  const dependsOnElectrs: { [key: string]: string[] } = newConfig?.[
+    "enable-electrs"
+  ]
+    ? { electrs: ["synced"] }
+    : {};
   const dependsOnBitcoind: { [key: string]: string[] } = newConfig?.txindex
     ? { bitcoind: [] }
     : {};
 
+    // add two const depsLnd and depsCln for the new lightning type string in getConfig
+  const depsLnd: { [key: string]: string[] } = newConfig?.lightning?.type === "lnd"  ? {lnd: []} : {};
+  const depsCln: { [key: string]: string[] } = newConfig?.lightning?.type === "cln"  ? {"c-lightning": []} : {};
+    
   return compat.setConfig(effects, newConfig, {
     ...dependsOnElectrs,
     ...dependsOnBitcoind,
+    ...depsLnd,
+    ...depsCln,
   });
 };