Mempool updated to use js procedures (#12)

* added js procs

* add js procs

* included dependencies in js procs

* modified dependencies.ts to remote fetch-blocks and fix proxy

* Update scripts/embassy.ts

Co-authored-by: J M <2364004+Blu-J@users.noreply.github.com>

* ignore .js files in script

* fixed typo

* Update scripts/services/setConfig.ts

Co-authored-by: J M <2364004+Blu-J@users.noreply.github.com>

* Update scripts/services/setConfig.ts

Co-authored-by: J M <2364004+Blu-J@users.noreply.github.com>

* fixed js dependencies

* return default if properties are not found

* await metadata

Co-authored-by: Dread <dread@start9labs.com>
Co-authored-by: J M <2364004+Blu-J@users.noreply.github.com>
Co-authored-by: Aiden McClelland <me@drbonez.dev>

Makefile

@@ -14,7 +14,7 @@ verify:  mempool.s9pk $(S9PK_PATH)
 install: mempool.s9pk
 	embassy-cli package install mempool.s9pk
 
-mempool.s9pk: manifest.yaml assets/* image.tar docs/instructions.md
+mempool.s9pk: manifest.yaml assets/* image.tar docs/instructions.md scripts/embassy.js 
 	embassy-sdk pack
 
 instructions.md: README.md
@@ -27,3 +27,6 @@ clean:
 	rm -f mempool.s9pk
 	rm -f image.tar
 
+scripts/embassy.js: scripts/**/*.ts
+	deno cache --reload scripts/embassy.ts
+	deno bundle scripts/embassy.ts scripts/embassy.js

manifest.yaml

@@ -1,7 +1,7 @@
 id: mempool
 title: Mempool
-version: 2.4.0
-release-notes: Update to Mempool v2.4.0 - added mining tab and additional reporting
+version: 2.4.0.1
+release-notes: Update to Mempool v2.4.0.1 - added js procedures for EOS performance improvements
 license: AGPL
 wrapper-repo: "https://github.com/Start9Labs/mempool-wrapper"
 upstream-repo: "https://github.com/mempool/mempool"
@@ -61,87 +61,23 @@ health-checks:
     inject: true
 config:
   get:
-    type: docker
-    image: compat
-    system: true
-    entrypoint: compat
-    args:
-      - config
-      - get
-      - /root
-      - "/mnt/assets/config_spec.yaml"
-    mounts:
-      compat: /mnt/assets
-      main: /root
-    io-format: yaml
+    type: script
   set:
-    type: docker
-    image: compat
-    system: true
-    entrypoint: compat
-    args:
-      - config
-      - set
-      - "mempool"
-      - /root
-      - "/mnt/assets/config_rules.yaml"
-      - "/mnt/assets/dependencies.yaml"
-    mounts:
-      compat: /mnt/assets
-      main: /root
-    io-format: yaml
+    type: script
 properties:
-  type: docker
-  image: compat
-  system: true
-  entrypoint: compat
-  args:
-    - properties
-    - /root
-  mounts:
-    main: /root
-    compat: /mnt/assets
-  io-format: yaml
+  type: script
 dependencies:
   bitcoind:
     version: ">=0.21.1.2 <24.0.0"
     requirement:
       type: "opt-out"
       how: Can alternatively configure proxy or bitcoin core  node.
-    description: Used to subscribe to new block events from a full archival node.
+    description: Used to subscribe to new block events from a full archival node
     config:
-      check: 
-        type: docker
-        image: compat
-        system: true
-        entrypoint: compat
-        args:
-          - dependency
-          - check
-          - "mempool"
-          - bitcoind
-          - /root
-          - "/mnt/assets/bitcoind_config_rules.yaml"
-        mounts:
-          compat: /mnt/assets
-          main: /root
-        io-format: yaml
+      check:
+        type: script
       auto-configure:
-        type: docker
-        image: compat
-        system: true
-        entrypoint: compat
-        args:
-          - dependency
-          - "auto-configure"
-          - "mempool"
-          - bitcoind
-          - /root
-          - "/mnt/assets/bitcoind_config_rules.yaml"
-        mounts:
-          compat: /mnt/assets
-          main: /root
-        io-format: yaml
+        type: script
   btc-rpc-proxy:
     version: ">=0.3.2.1 <0.4.0"
     requirement:
@@ -150,37 +86,9 @@ dependencies:
     description: Used for RPC permissioning.
     config:
       check:
-        type: docker
-        image: compat
-        system: true
-        entrypoint: compat
-        args:
-          - dependency
-          - check
-          - mempool
-          - "btc-rpc-proxy"
-          - /root
-          - "/mnt/assets/btc-rpc-proxy_config_rules.yaml"
-        mounts:
-          main: /root
-          compat: /mnt/assets
-        io-format: yaml
+        type: script
       auto-configure:
-        type: docker
-        image: compat
-        system: true
-        entrypoint: compat
-        args:
-          - dependency
-          - "auto-configure"
-          - mempool
-          - "btc-rpc-proxy"
-          - /root
-          - "/mnt/assets/btc-rpc-proxy_config_rules.yaml"
-        mounts:
-          main: /root
-          compat: /mnt/assets
-        io-format: yaml
+        type: script
   electrs:
     version: ">=0.9.6 <0.10.0"
     requirement:

migrations/eq_22_0_0.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -ea
+
+if [ $1 = "from" ]; then 
+    yq -i '.advanced.peers.addnode |= map(select(.hostname != ~))' /root/.bitcoin/start9/config.yaml
+    yq -i '.advanced.bloomfilters.peerbloomfilters = false' /root/.bitcoin/start9/config.yaml
+    echo '{"configured": true }'
+    exit 0
+elif [ $1 = "to" ]; then
+    yq -i 'del(.advanced.bloomfilters)' /root/.bitcoin/start9/config.yaml
+    echo '{"configured": false }'
+    exit 0
+else
+    echo "FATAL: Invalid argument: {from, to}. Migration failed." >&2
+    exit 1
+fi

migrations/gt_22_0_0_lt_23_0_0.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -ea
+
+if [ $1 = "from" ]; then 
+    yq -i '.advanced.peers.addnode |= map(select(.hostname != ~ or . == "*"))' /root/.bitcoin/start9/config.yaml
+    yq -i '.advanced.peers.addnode.[] |= {"hostname":., "port":~}' /root/.bitcoin/start9/config.yaml
+    yq -i '(.advanced.blockfilters.blockfilterindex, .advanced.blockfilters.peerblockfilters, .advanced.bloomfilters.peerbloomfilters) = false' /root/.bitcoin/start9/config.yaml
+    echo '{"configured": true }'
+    exit 0
+elif [ $1 = "to" ]; then
+    yq -i '.advanced.peers.addnode |= map_values(.hostname)' /root/.bitcoin/start9/config.yaml
+    yq -i 'del(.advanced.bloomfilters)' /root/.bitcoin/start9/config.yaml
+    echo '{"configured": false }'
+    exit 0
+else
+    echo "FATAL: Invalid argument: {from, to}. Migration failed." >&2
+    exit 1
+fi
+``

scripts/.gitignore

@@ -0,0 +1 @@
+*.js

scripts/deps.ts

@@ -0,0 +1,14 @@
+import matches from "https://deno.land/x/ts_matches@5.1.5/mod.ts";
+export * as YAML from "https://deno.land/std@0.140.0/encoding/yaml.ts";
+export type {
+  Config,
+  ConfigRes,
+  Effects,
+  ExpectedExports,
+  KnownError,
+  PackagePropertiesV2,
+  Properties,
+  SetResult,
+} from "https://start9.com/procedure/types.0.3.1.d.ts";
+
+export { matches };

scripts/embassy.ts

@@ -0,0 +1,4 @@
+export { setConfig } from "./services/setConfig.ts";
+export { properties } from "./services/properties.ts";
+export { getConfig } from "./services/getConfig.ts";
+export { dependencies } from "./services/dependencies.ts";

scripts/services/dependencies.ts

@@ -0,0 +1,157 @@
+import { ExpectedExports, Config, matches } from "../deps.ts";
+
+const { shape, arrayOf, string, boolean } = matches;
+
+const matchBitcoindConfig = shape({
+    advanced: shape({
+        pruning: shape({
+        mode: string,
+        }),
+    }),
+    rpc: shape({
+        enable: boolean,
+    }),
+    txindex: boolean,
+});
+
+const matchProxyConfig = shape({
+  users: arrayOf(
+    shape(
+      {
+        name: string,
+        "allowed-calls": arrayOf(string),
+        password: string,
+      },
+    )
+  ),
+});
+
+function times<T>(fn: (i: number) => T, amount: number): T[] {
+  const answer = new Array(amount);
+  for (let i = 0; i < amount; i++) {
+    answer[i] = fn(i);
+  }
+  return answer;
+}
+
+function randomItemString(input: string) {
+  return input[Math.floor(Math.random() * input.length)];
+}
+
+const serviceName = "mempool";
+const fullChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+type Check = {
+  currentError(config: Config): string | void;
+  fix(config: Config): void;
+};
+
+const proxyChecks: Array<Check> = [
+  {
+    currentError(config) {
+      if (!matchProxyConfig.test(config)) {
+        return "Config is not the correct shape";
+      }
+      if (config.users.some((x) => x.name === serviceName)) {
+        return;
+      }
+      return `Must have an RPC user named "${serviceName}"`;
+    },
+    fix(config) {
+      config.users.push({
+        name: serviceName,
+        "allowed-calls": [],
+        password: times(() => randomItemString(fullChars), 22).join(""),
+      });
+    },
+  },
+  ...[
+    "echo",
+    "getindexinfo",
+    "getblockcount",
+    "getchaintips",
+    "getmempoolinfo",
+    "getblockchaininfo",
+    "getblockhash",
+    "getblock",
+    "getmempoolentry",
+    "getrawtransaction",
+    "getrawmempool",
+    "gettxout",
+    "validateaddress",
+    "getblockstats",
+    "getnetworkhashps",
+    "getdifficulty",
+  ].map(
+    (operator): Check => ({
+      currentError(config) {
+        if (!matchProxyConfig.test(config)) {
+          return "Config is not the correct shape";
+        }
+        if (config.users.find((x) => x.name === serviceName)?.["allowed-calls"]?.some((x) => x === operator) ?? false) {
+          return;
+        }
+        return `RPC user "mempool" must have "${operator}" enabled`;
+      },
+      fix(config) {
+        if (!matchProxyConfig.test(config)) {
+          throw new Error("Config is not the correct shape");
+        }
+        const found = config.users.find((x) => x.name === serviceName);
+        if (!found) {
+          throw new Error("Users for mempool should exist");
+        }
+        found["allowed-calls"] = [...(found["allowed-calls"] ?? []), operator];
+      },
+    })
+  ),
+];
+
+export const dependencies: ExpectedExports.dependencies = {
+    bitcoind: {
+        async check(effects, configInput) {
+            effects.info("check bitcoind");
+            const config = matchBitcoindConfig.unsafeCast(configInput);
+            if (!config.rpc.enable) {
+                return { error: 'Must have RPC enabled' };
+            }
+            if (!config.txindex) {
+                return { error: 'Must have transaction indexing enabled' };
+            }
+            if (config.advanced.pruning.mode=="enabled") {
+                return { error: 'Pruning must be disabled to use Bitcoin Core directly. To use with a pruned node, set Bitcoin Core to Bitcoin Proxy instead.' };
+            }
+            return { result: null };
+        },
+        async autoConfigure(effects, configInput) {
+            effects.info("autoconfigure bitcoind");
+            const config = matchBitcoindConfig.unsafeCast(configInput);
+            config.rpc.enable = true;
+            config.txindex = true;
+            config.advanced.pruning.mode = "disabled";
+            return { result: config };
+        },
+    },
+    "btc-rpc-proxy": {
+        async check(effects, configInput) {
+            effects.info("check btc-rpc-proxy");
+            for (const checker of proxyChecks) {
+              const error = checker.currentError(configInput);
+              if (error) {
+                effects.error(`throwing error: ${error}`);
+                return { error };
+              }
+            }
+            return { result: null };
+        },
+        async autoConfigure(effects, configInput) {
+          effects.info("autoconfigure btc-rpc-proxy");
+          for (const checker of proxyChecks) {
+            const error = checker.currentError(configInput);
+            if (error) {
+              checker.fix(configInput);
+            }
+          }
+          return { result: configInput };
+        },
+    },
+};

scripts/services/getConfig.ts

@@ -0,0 +1,113 @@
+import { ConfigRes, ExpectedExports, matches, YAML } from "../deps.ts";
+
+const { any, string, dictionary } = matches;
+
+const matchConfig = dictionary([string, any]);
+
+export const getConfig: ExpectedExports.getConfig = async (effects) => {
+  const config = await effects
+    .readFile({
+      path: "start9/config.yaml",
+      volumeId: "main",
+    })
+    .then((x) => YAML.parse(x))
+    .then((x) => matchConfig.unsafeCast(x))
+    .catch((e) => {
+      effects.warn(`Got error ${e} while trying to read the config`);
+      return undefined;
+    });
+  const spec: ConfigRes["spec"] = {
+    "tor-address": {
+      "name": "Tor Address",
+      "description": "The Tor address of the network interface",
+      "type": "pointer",
+      "subtype": "package",
+      "package-id": "mempool",
+      "target": "tor-address",
+      "interface": "main",
+    },
+    "bitcoind": {
+      "type": "union",
+      "name": "Bitcoin Core",
+      "description": "The Bitcoin Core node to connect to:\n  - internal: The Bitcoin Core or Proxy services installed to your Embassy\n",
+      "tag": {
+        "id": "type",
+        "name": "Type",
+        "variant-names": {
+          "internal": "Bitcoin Core",
+          "internal-proxy": "Bitcoin Proxy"
+        },
+        "description": "The Bitcoin Core node to connect to:\n  - internal: The Bitcoin Core and Proxy services installed to your Embassy\n"
+      },
+      "default": "internal-proxy",
+      "variants": {
+        "internal": {
+          "user": {
+            "type": "pointer",
+            "name": "RPC Username",
+            "description": "The username for Bitcoin Core's RPC interface",
+            "subtype": "package",
+            "package-id": "bitcoind",
+            "target": "config",
+            "multi": false,
+            "selector": "$.rpc.username"
+          },
+          "password": {
+            "type": "pointer",
+            "name": "RPC Password",
+            "description": "The password for Bitcoin Core's RPC interface",
+            "subtype": "package",
+            "package-id": "bitcoind",
+            "target": "config",
+            "multi": false,
+            "selector": "$.rpc.password"
+          }
+        },
+        "internal-proxy": {
+          "user": {
+            "type": "pointer",
+            "name": "RPC Username",
+            "description": "The username for the RPC user allocated to electrs",
+            "subtype": "package",
+            "package-id": "btc-rpc-proxy",
+            "target": "config",
+            "multi": false,
+            "selector": "$.users[?(@.name == \"mempool\")].name"
+          },
+          "password": {
+            "type": "pointer",
+            "name": "RPC Password",
+            "description": "The password for the RPC user allocated to electrs",
+            "subtype": "package",
+            "package-id": "btc-rpc-proxy",
+            "target": "config",
+            "multi": false,
+            "selector": "$.users[?(@.name == \"mempool\")].password"
+          },
+          "txindex": {
+            "name": "Transaction Indexer",
+            "description": "The Transaction Indexer for Bitcoin Core",
+            "type": "pointer",
+            "subtype": "package",
+            "package-id": "bitcoind",
+            "target": "config",
+            "multi": false,
+            "selector": "$.txindex"
+          },
+        }
+      }
+    },
+    "enable-electrs": {
+      "name": "Enable Electrs Address Lookups",
+      "description": "Enables address lookups via an internal electrs instance",
+      "type": "boolean",
+      "default": true,
+    },
+  };
+  return {
+    result: {
+      config,
+      spec,
+    },
+  };
+};

scripts/services/properties.ts

@@ -0,0 +1,36 @@
+import { Effects, ExpectedExports, YAML } from "../deps.ts";
+
+export const properties: ExpectedExports.properties = async (
+  effects: Effects,
+) => {
+  try {
+    await effects.metadata({
+      path: "start9/stats.yaml",
+      volumeId: "main",
+    })
+  } catch {
+    return {
+      result: {
+        version: 2,
+        data: {
+          "Not Ready": {
+            type: "string",
+            value: "Could not find properties. The service might still be starting",
+            qr: false,
+            copyable: false,
+            masked: false,
+            description: "Fallback Message When Properties could not be found"
+          }
+        }
+      }
+    }
+  }
+  return {
+    result: YAML.parse(
+      await effects.readFile({
+        path: "start9/stats.yaml",
+        volumeId: "main",
+      }),
+    ) as any,
+  };
+};

scripts/services/setConfig.ts

@@ -0,0 +1,36 @@
+import {
+  Config,
+  Effects,
+  ExpectedExports,
+  matches,
+  SetResult,
+  YAML,
+} from "../deps.ts";
+const { number } = matches;
+
+export const setConfig: ExpectedExports.setConfig = async (
+  effects: Effects,
+  newConfig: Config,
+) => {
+  await effects.createDir({
+    path: "start9",
+    volumeId: "main",
+  });
+  await effects.writeFile({
+    path: "start9/config.yaml",
+    toWrite: YAML.stringify(newConfig),
+    volumeId: "main",
+  });
+  
+  const dependsOnElectrs: {[key: string]: string[]} = !!newConfig?.['enable-electrs'] ? {electrs: ['synced']} : {}
+  const dependsOnBitcoind: {[key: string]: string[]} =  !!newConfig?.txindex ? {bitcoind: []} : {}
+
+  const result: SetResult = {
+    signal: "SIGTERM",
+    "depends-on": {
+       ...dependsOnElectrs,
+       ...dependsOnBitcoind,
+    },
+  };
+  return { result };
+};