diff --git a/README.md b/README.md index 0eaa949..5a61c05 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ These tools are essential for a capable openclaw instance - screen capture, came | Tool | What it does | |------|--------------| | [**summarize**](https://github.com/steipete/summarize) | Link → clean text → summary | +| [**discrawl**](https://github.com/steipete/discrawl) | Mirror Discord into SQLite and search history locally | +| [**wacrawl**](https://github.com/steipete/wacrawl) | Read-only local archive and search for WhatsApp Desktop data | | [**gogcli**](https://github.com/steipete/gogcli) | Google CLI for Gmail, Calendar, Drive, and Contacts | | [**goplaces**](https://github.com/steipete/goplaces) | Google Places API (New) CLI | | [**camsnap**](https://github.com/steipete/camsnap) | Capture snapshots/clips from RTSP/ONVIF cameras | @@ -37,8 +39,10 @@ Each tool is a subflake under `tools//` exporting `openclawPlugin`. Point ```nix programs.openclaw.plugins = [ { source = "github:openclaw/nix-steipete-tools?dir=tools/camsnap"; } + { source = "github:openclaw/nix-steipete-tools?dir=tools/discrawl"; } { source = "github:openclaw/nix-steipete-tools?dir=tools/peekaboo"; } { source = "github:openclaw/nix-steipete-tools?dir=tools/summarize"; } + { source = "github:openclaw/nix-steipete-tools?dir=tools/wacrawl"; } ]; ``` @@ -56,13 +60,17 @@ inputs.nix-steipete-tools.url = "github:openclaw/nix-steipete-tools"; # Then use: inputs.nix-steipete-tools.packages.aarch64-darwin.camsnap +inputs.nix-steipete-tools.packages.aarch64-darwin.discrawl inputs.nix-steipete-tools.packages.aarch64-darwin.peekaboo +inputs.nix-steipete-tools.packages.aarch64-darwin.wacrawl # etc. # Linux examples: inputs.nix-steipete-tools.packages.x86_64-linux.camsnap +inputs.nix-steipete-tools.packages.x86_64-linux.discrawl inputs.nix-steipete-tools.packages.aarch64-linux.gogcli inputs.nix-steipete-tools.packages.x86_64-linux.summarize +inputs.nix-steipete-tools.packages.x86_64-linux.wacrawl ``` ## Skills syncing diff --git a/cmd/sync-skills/main.go b/cmd/sync-skills/main.go index 980871f..b4c1f91 100644 --- a/cmd/sync-skills/main.go +++ b/cmd/sync-skills/main.go @@ -60,6 +60,8 @@ func main() { mappings := []Mapping{ {"summarize", "skills/summarize"}, + {"discrawl", "skills/discrawl"}, + {"wacrawl", "skills/wacrawl"}, {"gogcli", "skills/gog"}, {"camsnap", "skills/camsnap"}, {"sonoscli", "skills/sonoscli"}, diff --git a/cmd/update-tools/main.go b/cmd/update-tools/main.go index f6e6f32..115c18a 100644 --- a/cmd/update-tools/main.go +++ b/cmd/update-tools/main.go @@ -147,6 +147,26 @@ func main() { } tools := []Tool{ + { + Name: "discrawl", + Repo: "steipete/discrawl", + Assets: []AssetSpec{ + {System: "aarch64-darwin", Regex: regexp.MustCompile(`discrawl_[0-9.]+_darwin_arm64\.tar\.gz`)}, + {System: "x86_64-linux", Regex: regexp.MustCompile(`discrawl_[0-9.]+_linux_amd64\.tar\.gz`)}, + {System: "aarch64-linux", Regex: regexp.MustCompile(`discrawl_[0-9.]+_linux_arm64\.tar\.gz`)}, + }, + NixFile: filepath.Join(repoRoot, "nix", "pkgs", "discrawl.nix"), + }, + { + Name: "wacrawl", + Repo: "steipete/wacrawl", + Assets: []AssetSpec{ + {System: "aarch64-darwin", Regex: regexp.MustCompile(`wacrawl_[0-9.]+_darwin_arm64\.tar\.gz`)}, + {System: "x86_64-linux", Regex: regexp.MustCompile(`wacrawl_[0-9.]+_linux_amd64\.tar\.gz`)}, + {System: "aarch64-linux", Regex: regexp.MustCompile(`wacrawl_[0-9.]+_linux_arm64\.tar\.gz`)}, + }, + NixFile: filepath.Join(repoRoot, "nix", "pkgs", "wacrawl.nix"), + }, { Name: "gogcli", Repo: "steipete/gogcli", diff --git a/flake.nix b/flake.nix index 6fef5d9..08241bb 100644 --- a/flake.nix +++ b/flake.nix @@ -12,6 +12,8 @@ forAllSystems = f: lib.genAttrs systems (system: f system); packageSystems = { summarize = [ "aarch64-darwin" "x86_64-linux" "aarch64-linux" ]; + discrawl = [ "aarch64-darwin" "x86_64-linux" "aarch64-linux" ]; + wacrawl = [ "aarch64-darwin" "x86_64-linux" "aarch64-linux" ]; gogcli = [ "aarch64-darwin" "x86_64-linux" "aarch64-linux" ]; goplaces = [ "aarch64-darwin" "x86_64-linux" "aarch64-linux" ]; camsnap = [ "aarch64-darwin" "x86_64-linux" "aarch64-linux" ]; @@ -34,6 +36,12 @@ nodejs = if pkgs ? nodejs_22 then pkgs.nodejs_22 else pkgs.nodejs; }; }) + // (lib.optionalAttrs (supports "discrawl") { + discrawl = pkgs.callPackage ./nix/pkgs/discrawl.nix {}; + }) + // (lib.optionalAttrs (supports "wacrawl") { + wacrawl = pkgs.callPackage ./nix/pkgs/wacrawl.nix {}; + }) // (lib.optionalAttrs (supports "gogcli") { gogcli = pkgs.callPackage ./nix/pkgs/gogcli.nix {}; }) diff --git a/nix/pkgs/discrawl.nix b/nix/pkgs/discrawl.nix new file mode 100644 index 0000000..493496e --- /dev/null +++ b/nix/pkgs/discrawl.nix @@ -0,0 +1,53 @@ +{ lib, stdenv, fetchurl }: + +let + sources = { + "aarch64-darwin" = { + url = "https://github.com/steipete/discrawl/releases/download/v0.6.0/discrawl_0.6.0_darwin_arm64.tar.gz"; + hash = "sha256-VAGD2mmbZP3pxBgtzg9n3DE89NDLKD5iUIVMbJYwaOo="; + }; + "x86_64-linux" = { + url = "https://github.com/steipete/discrawl/releases/download/v0.6.0/discrawl_0.6.0_linux_amd64.tar.gz"; + hash = "sha256-KQlpw/I1asBK4+3IlbCiO76+ysJIMVSSuwqrr9UG190="; + }; + "aarch64-linux" = { + url = "https://github.com/steipete/discrawl/releases/download/v0.6.0/discrawl_0.6.0_linux_arm64.tar.gz"; + hash = "sha256-2uRQonrM4THiSyNUir9bgOyaTWHbPv4brIPCNW4yU4s="; + }; + }; +in +stdenv.mkDerivation { + pname = "discrawl"; + version = "0.6.0"; + + src = fetchurl sources.${stdenv.hostPlatform.system}; + + dontConfigure = true; + dontBuild = true; + + unpackPhase = '' + tar -xzf "$src" + ''; + + installPhase = '' + runHook preInstall + mkdir -p "$out/bin" "$out/share/doc/discrawl" + cp $(find . -type f -name discrawl | head -1) "$out/bin/discrawl" + chmod 0755 "$out/bin/discrawl" + if [ -f LICENSE ]; then + cp LICENSE "$out/share/doc/discrawl/" + fi + if [ -f README.md ]; then + cp README.md "$out/share/doc/discrawl/" + fi + runHook postInstall + ''; + + meta = with lib; { + description = "Mirror Discord into SQLite and search server history locally"; + homepage = "https://github.com/steipete/discrawl"; + license = licenses.mit; + platforms = builtins.attrNames sources; + mainProgram = "discrawl"; + }; +} diff --git a/nix/pkgs/wacrawl.nix b/nix/pkgs/wacrawl.nix new file mode 100644 index 0000000..2e12fd9 --- /dev/null +++ b/nix/pkgs/wacrawl.nix @@ -0,0 +1,53 @@ +{ lib, stdenv, fetchurl }: + +let + sources = { + "aarch64-darwin" = { + url = "https://github.com/steipete/wacrawl/releases/download/v0.1.0/wacrawl_0.1.0_darwin_arm64.tar.gz"; + hash = "sha256-GJw2RODtrn1uOdduq9lSYAWzXVkSQPMBuiweyCwov6g="; + }; + "x86_64-linux" = { + url = "https://github.com/steipete/wacrawl/releases/download/v0.1.0/wacrawl_0.1.0_linux_amd64.tar.gz"; + hash = "sha256-cMNmH/Xj++5dAIiphVz7e7DYDwM2xx+RlEVxBKw7kJw="; + }; + "aarch64-linux" = { + url = "https://github.com/steipete/wacrawl/releases/download/v0.1.0/wacrawl_0.1.0_linux_arm64.tar.gz"; + hash = "sha256-QybKaF1lmqGVDUwqBLWfEtJwkT/BB9KsJgL1a/m40hQ="; + }; + }; +in +stdenv.mkDerivation { + pname = "wacrawl"; + version = "0.1.0"; + + src = fetchurl sources.${stdenv.hostPlatform.system}; + + dontConfigure = true; + dontBuild = true; + + unpackPhase = '' + tar -xzf "$src" + ''; + + installPhase = '' + runHook preInstall + mkdir -p "$out/bin" "$out/share/doc/wacrawl" + cp $(find . -type f -name wacrawl | head -1) "$out/bin/wacrawl" + chmod 0755 "$out/bin/wacrawl" + if [ -f LICENSE ]; then + cp LICENSE "$out/share/doc/wacrawl/" + fi + if [ -f README.md ]; then + cp README.md "$out/share/doc/wacrawl/" + fi + runHook postInstall + ''; + + meta = with lib; { + description = "Read-only local archive and search for WhatsApp Desktop data"; + homepage = "https://github.com/steipete/wacrawl"; + license = licenses.mit; + platforms = builtins.attrNames sources; + mainProgram = "wacrawl"; + }; +} diff --git a/tools/discrawl/flake.nix b/tools/discrawl/flake.nix new file mode 100644 index 0000000..d7ad721 --- /dev/null +++ b/tools/discrawl/flake.nix @@ -0,0 +1,38 @@ +{ + description = "openclaw plugin: discrawl"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs?rev=16c7794d0a28b5a37904d55bcca36003b9109aaa&narHash=sha256-fFUnEYMla8b7UKjijLnMe%2BoVFOz6HjijGGNS1l7dYaQ%3D"; + root.url = "path:../.."; + }; + + outputs = { self, nixpkgs, root }: + let + lib = nixpkgs.lib; + systems = builtins.attrNames root.packages; + pluginFor = system: + let + packagesForSystem = root.packages.${system} or {}; + discrawl = packagesForSystem.discrawl or null; + in + if discrawl == null then null else { + name = "discrawl"; + skills = [ ./skills/discrawl ]; + packages = [ discrawl ]; + needs = { + stateDirs = [ ".discrawl" ]; + requiredEnv = [ ]; + }; + }; + in { + packages = lib.genAttrs systems (system: + let + discrawl = (root.packages.${system} or {}).discrawl or null; + in + if discrawl == null then {} + else { discrawl = discrawl; } + ); + + openclawPlugin = pluginFor; + }; +} diff --git a/tools/discrawl/skills/discrawl/SKILL.md b/tools/discrawl/skills/discrawl/SKILL.md new file mode 100644 index 0000000..7892092 --- /dev/null +++ b/tools/discrawl/skills/discrawl/SKILL.md @@ -0,0 +1,74 @@ +--- +name: discrawl +description: Mirror Discord guild history into local SQLite and query it offline with search, messages, mentions, reports, and DM wiretap import. +homepage: https://github.com/steipete/discrawl +metadata: + { + "openclaw": + { + "emoji": "🛰️", + "requires": { "bins": ["discrawl"] }, + "install": + [ + { + "id": "brew", + "kind": "brew", + "formula": "steipete/tap/discrawl", + "bins": ["discrawl"], + "label": "Install discrawl (brew)", + }, + ], + }, + } +--- + +# discrawl + +Use `discrawl` to mirror Discord guild data into local SQLite, then query it offline. + +## When to Use + +Use this skill when the user wants to: + +- search Discord history locally without relying on Discord search +- archive a guild into SQLite for later queries +- inspect recent messages, mentions, channels, or members from a local archive +- import local Discord Desktop cache data for DM recovery/search +- publish or subscribe to a Git-backed Discord archive snapshot + +## Requirements + +- Discord bot token for guild sync, or an existing OpenClaw Discord config +- local Discord Desktop cache files only if using `wiretap` +- enough local disk for SQLite archive growth + +## Setup + +- Default config: `~/.discrawl/config.toml` +- Default database: `~/.discrawl/discrawl.db` +- Fastest setup when OpenClaw already has Discord configured: + - `discrawl init --from-openclaw ~/.openclaw/openclaw.json` +- Env-only setup: + - `export DISCORD_BOT_TOKEN="..."` + - `discrawl init` + +## Common Commands + +- Doctor: `discrawl doctor` +- Initial history: `discrawl sync --full` +- Incremental refresh: `discrawl sync` +- Live tail: `discrawl tail` +- Search: `discrawl search "panic nil pointer"` +- Recent channel messages: `discrawl messages --channel general --hours 24` +- Mentions: `discrawl mentions --user ` +- DM cache import: `discrawl wiretap` +- Local DM search: `discrawl dms --search "launch checklist"` +- Read-only SQL: `discrawl sql "select count(*) from messages"` +- Git-backed reader mode: `discrawl subscribe ` + +## Notes + +- Bot-token sync reads only guilds/channels the bot can access. +- `wiretap` uses local Discord Desktop cache files only; it does not use a user token. +- Prefer `discrawl doctor` before a first sync. +- Use `sync --full` once for backfill, then plain `sync` for routine refreshes. diff --git a/tools/wacrawl/flake.nix b/tools/wacrawl/flake.nix new file mode 100644 index 0000000..4b1b539 --- /dev/null +++ b/tools/wacrawl/flake.nix @@ -0,0 +1,38 @@ +{ + description = "openclaw plugin: wacrawl"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs?rev=16c7794d0a28b5a37904d55bcca36003b9109aaa&narHash=sha256-fFUnEYMla8b7UKjijLnMe%2BoVFOz6HjijGGNS1l7dYaQ%3D"; + root.url = "path:../.."; + }; + + outputs = { self, nixpkgs, root }: + let + lib = nixpkgs.lib; + systems = builtins.attrNames root.packages; + pluginFor = system: + let + packagesForSystem = root.packages.${system} or {}; + wacrawl = packagesForSystem.wacrawl or null; + in + if wacrawl == null then null else { + name = "wacrawl"; + skills = [ ./skills/wacrawl ]; + packages = [ wacrawl ]; + needs = { + stateDirs = [ ".wacrawl" ]; + requiredEnv = [ ]; + }; + }; + in { + packages = lib.genAttrs systems (system: + let + wacrawl = (root.packages.${system} or {}).wacrawl or null; + in + if wacrawl == null then {} + else { wacrawl = wacrawl; } + ); + + openclawPlugin = pluginFor; + }; +} diff --git a/tools/wacrawl/skills/wacrawl/SKILL.md b/tools/wacrawl/skills/wacrawl/SKILL.md new file mode 100644 index 0000000..497dfcd --- /dev/null +++ b/tools/wacrawl/skills/wacrawl/SKILL.md @@ -0,0 +1,69 @@ +--- +name: wacrawl +description: Read-only local archive and search for WhatsApp Desktop chats, messages, and media metadata. +homepage: https://github.com/steipete/wacrawl +metadata: + { + "openclaw": + { + "emoji": "💬", + "requires": { "bins": ["wacrawl"] }, + "install": + [ + { + "id": "brew", + "kind": "brew", + "formula": "steipete/tap/wacrawl", + "bins": ["wacrawl"], + "label": "Install wacrawl (brew)", + }, + ], + }, + } +--- + +# wacrawl + +Use `wacrawl` to snapshot local WhatsApp Desktop data into a separate SQLite archive and search it offline. + +## When to Use + +Use this skill when the user wants to: + +- inspect local WhatsApp Desktop history without opening the app +- archive chats into a local SQLite database for repeat queries +- search WhatsApp messages locally with filters +- list chats, recent messages, or archive status from a read-only import + +## Requirements + +- local WhatsApp Desktop data on the same machine +- enough local disk for `~/.wacrawl/wacrawl.db` +- understand that this is read-only inspection, not message sending + +## Setup + +- Default source: `~/Library/Group Containers/group.net.whatsapp.WhatsApp.shared` +- Default archive DB: `~/.wacrawl/wacrawl.db` +- First sanity check: + - `wacrawl doctor` +- First import: + - `wacrawl import` + +## Common Commands + +- Doctor: `wacrawl doctor` +- Import fresh snapshot: `wacrawl import` +- Archive status: `wacrawl status` +- List chats: `wacrawl chats --limit 20` +- Recent messages: `wacrawl messages --limit 20` +- One chat: `wacrawl messages --chat 1234567890@s.whatsapp.net --limit 50` +- Search: `wacrawl search "release notes"` +- Filtered search: `wacrawl --json search "invoice" --from-them --after 2026-01-01` + +## Notes + +- `wacrawl` is read-only and does not send messages. +- It copies WhatsApp SQLite files into a temp snapshot before import. +- Use `--source` to override the WhatsApp Desktop container path. +- Use `--db` to archive somewhere other than `~/.wacrawl/wacrawl.db`.