openclaw/discrawl
openclaw/discrawl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases 4 Wiki Activity

refactor: remove OpenClaw-specific config coupling

Browse Source
This commit is contained in:
Peter Steinberger 2026-04-27 20:39:35 +01:00
parent 6808268342
commit 7af036009e
No known key found for this signature in database
16 changed files with 114 additions and 1760 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
.github/workflows/discord-backup-report.yml vendored
View File

@ -26,11 +26,6 @@ jobs:
go-version-file: go.mod
cache: true
- name: Setup Node
uses: actions/setup-node@v6.4.0
with:
node-version: "24"
- name: Restore Discord DB cache
id: restore-discord-db
uses: actions/cache/restore@v5.0.5
@ -51,46 +46,27 @@ jobs:
- name: Generate daily Discord report
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
DISCORD_BACKUP_TOKEN: ${{ secrets.DISCORD_BACKUP_TOKEN }}
GH_TOKEN: ${{ secrets.DISCORD_FIELD_NOTES_GITHUB_TOKEN || github.token }}
DISCRAWL_BACKUP_REPOSITORY: ${{ secrets.DISCRAWL_BACKUP_REPOSITORY }}
CONFIG: ${{ runner.temp }}/discrawl/config.toml
DB: ${{ github.workspace }}/.discrawl-ci/discrawl.db
BACKUP_REPO: ${{ runner.temp }}/discord-backup
OPENCLAW_STATE_DIR: ${{ runner.temp }}/openclaw
DISCORD_FIELD_NOTES_GITHUB_REPO: openclaw/openclaw
run: |
if [ -z "${DISCORD_BACKUP_TOKEN:-}" ]; then
echo "::error title=Missing secret::Configure DISCORD_BACKUP_TOKEN with write access to openclaw/discord-backup."
echo "::error title=Missing secret::Configure DISCORD_BACKUP_TOKEN with write access to the backup repository."
exit 1
fi
BACKUP_REMOTE="https://x-access-token:${DISCORD_BACKUP_TOKEN}@github.com/openclaw/discord-backup.git"
if [ -z "${DISCRAWL_BACKUP_REPOSITORY:-}" ]; then
echo "::error title=Missing secret::Configure DISCRAWL_BACKUP_REPOSITORY as owner/repo for the backup repository."
exit 1
fi
BACKUP_REMOTE="https://x-access-token:${DISCORD_BACKUP_TOKEN}@github.com/${DISCRAWL_BACKUP_REPOSITORY}.git"
mkdir -p "$(dirname "$CONFIG")"
mkdir -p "$(dirname "$DB")"
git clone "$BACKUP_REMOTE" "$BACKUP_REPO"
printf 'db_path = "%s"\n' "$DB" > "$CONFIG"
go run ./cmd/discrawl --config "$CONFIG" subscribe --repo "$BACKUP_REPO" "$BACKUP_REMOTE"
go run ./cmd/discrawl --config "$CONFIG" report --readme "$BACKUP_REPO/README.md"
if [ -n "${OPENAI_API_KEY:-}" ]; then
npm install -g openclaw@latest
openclaw onboard \
--non-interactive \
--mode local \
--auth-choice openai-api-key \
--secret-input-mode ref \
--accept-risk \
--skip-daemon \
--skip-skills \
--skip-search \
--skip-health
tmp_config="$(mktemp)"
jq '.agents.defaults.model = "openai/gpt-5.2" | .agents.defaults.timeoutSeconds = 300 | .agents.defaults.llm.idleTimeoutSeconds = 240' \
"$OPENCLAW_STATE_DIR/openclaw.json" > "$tmp_config"
mv "$tmp_config" "$OPENCLAW_STATE_DIR/openclaw.json"
scripts/discord-backup-field-notes.sh "$CONFIG" "$BACKUP_REPO"
else
echo "OPENAI_API_KEY not configured; skipping OpenClaw field notes"
fi
if git -C "$BACKUP_REPO" diff --quiet README.md; then
echo "README already up to date"
exit 0

9
.github/workflows/publish-discord-backup.yml vendored
View File

@ -48,6 +48,7 @@ jobs:
env:
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
DISCORD_BACKUP_TOKEN: ${{ secrets.DISCORD_BACKUP_TOKEN }}
DISCRAWL_BACKUP_REPOSITORY: ${{ secrets.DISCRAWL_BACKUP_REPOSITORY }}
DISCRAWL_GUILD_ID: ${{ secrets.DISCRAWL_GUILD_ID }}
CONFIG: ${{ runner.temp }}/discrawl/config.toml
DB: ${{ github.workspace }}/.discrawl-ci/discrawl.db
@ -58,14 +59,18 @@ jobs:
exit 1
fi
if [ -z "${DISCORD_BACKUP_TOKEN:-}" ]; then
echo "::error title=Missing secret::Configure DISCORD_BACKUP_TOKEN with write access to openclaw/discord-backup."
echo "::error title=Missing secret::Configure DISCORD_BACKUP_TOKEN with write access to the backup repository."
exit 1
fi
if [ -z "${DISCRAWL_BACKUP_REPOSITORY:-}" ]; then
echo "::error title=Missing secret::Configure DISCRAWL_BACKUP_REPOSITORY as owner/repo for the backup repository."
exit 1
fi
if [ -z "${DISCRAWL_GUILD_ID:-}" ]; then
echo "::error title=Missing secret::Configure DISCRAWL_GUILD_ID with the Discord guild to publish."
exit 1
fi
BACKUP_REMOTE="https://x-access-token:${DISCORD_BACKUP_TOKEN}@github.com/openclaw/discord-backup.git"
BACKUP_REMOTE="https://x-access-token:${DISCORD_BACKUP_TOKEN}@github.com/${DISCRAWL_BACKUP_REPOSITORY}.git"
mkdir -p "$(dirname "$CONFIG")"
mkdir -p "$(dirname "$DB")"
git clone "$BACKUP_REMOTE" "$BACKUP_REPO"

9
CHANGELOG.md
View File

@ -9,10 +9,6 @@ All notable changes to `discrawl` will be documented in this file.
- Refreshed Go module dependencies and CI tool/action pins, including staticcheck, gofumpt, gosec, govulncheck, gitleaks, setup-node, and GoReleaser.
- Hardened report README writes and Discord Desktop cache reads with root-scoped filesystem access to satisfy the latest gosec checks.
### Fixes
- OpenClaw Discord token loading now accepts SecretRef objects backed by file or env providers in addition to plaintext token strings, with shared resolver coverage for provider defaults, allowlists, empty values, and unsupported exec refs. (#49) Thanks @TeodoroRodrigo.
## 0.6.0 - 2026-04-24
### Changes
@ -84,11 +80,10 @@ All notable changes to `discrawl` will be documented in this file.
- `sync --all` now bypasses `default_guild_id` so one run can fan out across every discovered guild without clearing the single-guild default first
- `sync --full` no longer aborts when forum thread discovery hits Discord `403 Missing Access`; inaccessible channels are skipped and marked unavailable while accessible channels continue syncing
- startup now validates and stamps SQLite schema version via `PRAGMA user_version`, and fails fast if the local DB schema is newer than the running binary
- `init --from-openclaw` now supports `--account`, and OpenClaw token fields can use `${ENV_VAR}` placeholders
- git-backed archive sharing can now export/import compressed JSONL snapshots with manifests, subscribe to a Git repo as the data source, and run in git-only mode without Discord credentials
- `messages`, `search`, and reports can automatically refresh stale git-backed data, preferring the Git snapshot before falling back to live Discord when both sources are configured
- the Discord backup publisher workflow now syncs latest messages, publishes the archive to a private GitHub repo, serializes concurrent runs, validates required secrets, and skips the member crawl for faster updates
- the backup report workflow now updates README activity stats, supports OpenClaw-generated field notes, runs the field-note logic from the backup action, and keeps those queries bounded with process timeouts
- the backup report workflow now updates README activity stats from the backup action and keeps those queries bounded with process timeouts
- `sync --latest-only` adds a lightweight refresh path for checking recent Discord messages without doing a full historical crawl
- repository imports now skip expensive rebuilds when the snapshot manifest is already current, and GitHub Actions persist the warmed SQLite database across runs
- the Docker git-source smoke test now verifies that a fresh install can subscribe to a repository-only archive and query messages, SQL, and reports
@ -112,7 +107,7 @@ All notable changes to `discrawl` will be documented in this file.
- multi-guild Discord crawler with single-guild default UX
- local SQLite archive with FTS5 search
- commands: `init`, `sync`, `tail`, `search`, `messages`, `mentions`, `sql`, `members`, `channels`, `status`, `doctor`
- OpenClaw config reuse plus env-based bot token discovery
- env-based bot token discovery
- resumable full-history sync, live gateway tailing, repair sync loop, targeted channel sync
- attachment-text indexing for small text-like uploads
- structured user and role mention indexing/querying

59
README.md
View File

@ -59,12 +59,11 @@ Without those intents/permissions, `sync`, `tail`, member snapshots, or message
Token resolution:
1. OpenClaw config, if `discord.token_source` is not `env`
2. `DISCORD_BOT_TOKEN` or the configured `discord.token_env`
1. `DISCORD_BOT_TOKEN` or the configured `discord.token_env`
`discrawl` accepts either raw token text or a value prefixed with `Bot `. It normalizes that automatically.
Fastest env-only path:
Fastest path:
```bash
export DISCORD_BOT_TOKEN="your-bot-token"
@ -80,8 +79,6 @@ export DISCORD_BOT_TOKEN="your-bot-token"
Then reload your shell before running `discrawl`.
If you already use OpenClaw, `discrawl` can reuse the Discord token from `~/.openclaw/openclaw.json` by default.
Default runtime paths:
- config: `~/.discrawl/config.toml`
@ -111,10 +108,11 @@ Examples below assume `discrawl` is on `PATH`. If you built from source without
## Quick Start
Reuse an existing OpenClaw Discord bot config and refresh both bot-visible guild data and local desktop cache data:
Configure a Discord bot token and refresh both bot-visible guild data and local desktop cache data:
```bash
discrawl init --from-openclaw ~/.openclaw/openclaw.json
export DISCORD_BOT_TOKEN="..."
discrawl init
discrawl doctor
discrawl sync --full
discrawl sync
@ -124,26 +122,10 @@ discrawl tail
Use `discrawl sync --source wiretap` when you only want the local Discord Desktop cache import and do not want bot-token API sync.
Multi-account OpenClaw setup:
```bash
discrawl init --from-openclaw ~/.openclaw/openclaw.json --account atlas
```
Env-only setup:
```bash
export DISCORD_BOT_TOKEN="..."
discrawl doctor
discrawl init
discrawl sync --full
discrawl sync
```
Git-only reader setup:
```bash
discrawl subscribe https://github.com/openclaw/discord-backup.git
discrawl subscribe https://github.com/example/discord-archive.git
discrawl search "launch checklist"
discrawl messages --channel general --hours 24
```
@ -167,14 +149,10 @@ Creates the local config and discovers accessible guilds.
```bash
discrawl init
discrawl init --from-openclaw ~/.openclaw/openclaw.json
discrawl init --from-openclaw ~/.openclaw/openclaw.json --account atlas
discrawl init --guild 123456789012345678
discrawl init --db ~/data/discrawl.db
```
When OpenClaw config tokens use `${ENV_VAR}` placeholders, `init` and `doctor` resolve them before auth.
### `sync`
Refreshes SQLite from one or both archive sources.
@ -437,14 +415,14 @@ discrawl status
Publisher:
```bash
discrawl publish --remote https://github.com/openclaw/discord-backup.git --push
discrawl publish --remote https://github.com/example/discord-archive.git --push
discrawl publish --readme path/to/discord-backup/README.md --push
```
Subscriber:
```bash
discrawl subscribe https://github.com/openclaw/discord-backup.git
discrawl subscribe https://github.com/example/discord-archive.git
discrawl search "launch checklist"
discrawl messages --channel general --hours 24
```
@ -454,8 +432,8 @@ discrawl messages --channel general --hours 24
Configure freshness:
```bash
discrawl subscribe --stale-after 15m https://github.com/openclaw/discord-backup.git
discrawl subscribe --no-auto-update https://github.com/openclaw/discord-backup.git
discrawl subscribe --stale-after 15m https://github.com/example/discord-archive.git
discrawl subscribe --no-auto-update https://github.com/example/discord-archive.git
```
Once `share.remote` is configured, read commands auto-fetch and import when the local share import is older than `share.stale_after` (default `15m`). `discrawl update` forces the same pull/import step manually.
@ -468,7 +446,7 @@ Generated vectors can be backed up explicitly:
```bash
discrawl publish --with-embeddings --push
discrawl subscribe --with-embeddings https://github.com/openclaw/discord-backup.git
discrawl subscribe --with-embeddings https://github.com/example/discord-archive.git
discrawl update --with-embeddings
```
@ -491,15 +469,6 @@ discrawl report --readme path/to/discord-backup/README.md
Every scheduled snapshot publish updates deterministic README stats: latest update time, latest archived message, archive totals, and day/week/month activity.
The backup README field notes are intentionally a separate daily workflow, not part of `discrawl report`, so model latency or quota cannot block the 15-minute data publish path. `.github/workflows/discord-backup-report.yml` installs `openclaw@latest`, runs `openclaw agent --local` with OpenAI, and inserts a separate `discrawl-field-notes` block with:
- what people seem to love
- what people complain about
- complaint topics correlated with recent GitHub issue and PR clusters
- the likely best PR to watch
Configure `OPENAI_API_KEY` in the discrawl repo secrets to enable agent-written field notes. `DISCORD_BACKUP_TOKEN` still needs write access to `openclaw/discord-backup`. If the GitHub repo used for issue/PR correlation is private, also set `DISCORD_FIELD_NOTES_GITHUB_TOKEN` with read access to that repo.
The backup workflows restore and save `.discrawl-ci/discrawl.db` with `actions/cache`. On a warm runner cache, `discrawl update` compares the cached DB's last imported snapshot timestamp with `manifest.json` and skips the full sharded import when they match. Cache misses and newer backup manifests still take the normal pull/import path.
### `doctor`
@ -525,9 +494,7 @@ cache_dir = "~/.discrawl/cache"
log_dir = "~/.discrawl/logs"
[discord]
token_source = "openclaw" # use "none" for Git-only read access
openclaw_config = "~/.openclaw/openclaw.json"
account = "default"
token_source = "env" # use "none" for Git-only read access
token_env = "DISCORD_BOT_TOKEN"
[sync]
@ -565,7 +532,7 @@ Config override rules:
- `--config` beats everything
- `DISCRAWL_CONFIG` overrides the default config path
- `discord.token_source = "env"` forces env-only token lookup
- `discord.token_source = "none"` disables live Discord access for Git-only readers
- `DISCRAWL_NO_AUTO_UPDATE=1` disables Git snapshot auto-update for read commands in one process, useful for report jobs that already imported a fresh backup.
## Embeddings

36
SPEC.md
View File

@ -51,7 +51,7 @@ These are settled unless the user explicitly changes them:
- DB location: `~/.discrawl/discrawl.db`
- cache dir: `~/.discrawl/cache/`
- log dir: `~/.discrawl/logs/`
- token source: reuse Molty / existing OpenClaw Discord bot config
- token source: `DISCORD_BOT_TOKEN` or configured env var
- guild model: one guild in CLI UX, multi-guild-ready schema
- search: hybrid, with FTS first and embeddings optional
- embedding provider: OpenAI
@ -71,33 +71,12 @@ An agent should assume:
- Go is installed and modern
- user is Peter
- user keeps many secrets in `~/.profile`
- an existing OpenClaw install may already contain usable Discord bot config
### Key file paths
- `~/.discrawl/config.toml`
- `~/.discrawl/discrawl.db`
- `~/.profile`
- `~/.openclaw/openclaw.json`
- `~/.openclaw/openclaw.json.bak*`
### Existing bot config
The current bot token source is expected in:
- `~/.openclaw/openclaw.json`
Expected path inside JSON:
- `channels.discord.token`
Expected guild selection path:
- `channels.discord.guilds`
The current intended default mode is:
- `discrawl init --from-openclaw ~/.openclaw/openclaw.json`
### OpenAI embeddings key
@ -428,12 +407,11 @@ discrawl [global flags] <command> [args]
Purpose:
- create `~/.discrawl/config.toml`
- import defaults from OpenClaw
- discover accessible Discord guilds
- persist guild id and DB path
Expected flags:
- `--from-openclaw <path>`
- `--guild <id>`
- `--db <path>`
- `--with-embeddings`
@ -556,7 +534,7 @@ Must show:
Must check:
- config file readable
- OpenClaw token source readable
- Discord token env var readable unless live access is disabled
- Discord auth valid
- guild reachable
- DB openable
@ -583,9 +561,8 @@ cache_dir = "~/.discrawl/cache"
log_dir = "~/.discrawl/logs"
[discord]
token_source = "openclaw"
openclaw_config = "~/.openclaw/openclaw.json"
channel_account = "discord"
token_source = "env"
token_env = "DISCORD_BOT_TOKEN"
[sync]
concurrency = 4
@ -612,6 +589,7 @@ Config precedence:
Environment variables:
- `DISCRAWL_CONFIG`
- `DISCORD_BOT_TOKEN`
- `OPENAI_API_KEY`
## Token Handling Rules
@ -624,7 +602,7 @@ Do not:
Do:
- load bot token from OpenClaw config path
- load bot token from env
- load OpenAI key from env
- redact secrets in debug and doctor output

12
internal/cli/admin_commands.go
View File

@ -36,8 +36,6 @@ type syncRunStats struct {
func (r *runtime) runInit(args []string) error {
fs := flag.NewFlagSet("init", flag.ContinueOnError)
fs.SetOutput(io.Discard)
fromOpenClaw := fs.String("from-openclaw", "", "")
account := fs.String("account", "", "")
guildID := fs.String("guild", "", "")
dbPath := fs.String("db", "", "")
withEmbeddings := fs.Bool("with-embeddings", false, "")
@ -45,12 +43,6 @@ func (r *runtime) runInit(args []string) error {
return usageErr(err)
}
cfg := config.Default()
if *fromOpenClaw != "" {
cfg.Discord.OpenClawConfig = *fromOpenClaw
}
if *account != "" {
cfg.Discord.Account = *account
}
if *dbPath != "" {
cfg.DBPath = *dbPath
}
@ -90,10 +82,6 @@ func (r *runtime) runInit(args []string) error {
}
if *guildID != "" {
cfg.DefaultGuildID = *guildID
} else if info, err := config.LoadOpenClawDiscord(cfg.Discord.OpenClawConfig, cfg.Discord.Account); err == nil {
if len(info.GuildIDs) == 1 {
cfg.DefaultGuildID = info.GuildIDs[0]
}
}
if cfg.DefaultGuildID == "" && len(cfg.GuildIDs) == 1 {
cfg.DefaultGuildID = cfg.GuildIDs[0]

10
internal/cli/cli_test.go
View File

@ -1207,13 +1207,12 @@ func TestRuntimeInitSyncTailAndDoctor(t *testing.T) {
}
rt := newRuntime()
require.NoError(t, rt.runInit([]string{"--db", dbPath, "--with-embeddings", "--guild", "g2", "--account", "atlas"}))
require.NoError(t, rt.runInit([]string{"--db", dbPath, "--with-embeddings", "--guild", "g2"}))
cfg, err := config.Load(cfgPath)
require.NoError(t, err)
require.Equal(t, []string{"g1", "g2"}, cfg.GuildIDs)
require.Equal(t, "g2", cfg.DefaultGuildID)
require.Equal(t, "atlas", cfg.Discord.Account)
require.True(t, cfg.Search.Embeddings.Enabled)
cfg.Desktop.Path = filepath.Join(dir, "empty-discord")
require.NoError(t, os.MkdirAll(cfg.Desktop.Path, 0o755))
@ -1457,7 +1456,8 @@ func TestCommandUsageBranches(t *testing.T) {
{[]string{"--config", cfgPath, "embed", "--batch-size", "0"}, "--batch-size must be positive"},
{[]string{"--config", cfgPath, "publish", "extra"}, "publish takes no positional arguments"},
{[]string{"--config", cfgPath, "update", "extra"}, "update takes no positional arguments"},
{[]string{"--config", cfgPath, "subscribe", "one", "two"}, "subscribe takes at most one remote"},
{[]string{"--config", cfgPath, "subscribe"}, "subscribe requires one remote"},
{[]string{"--config", cfgPath, "subscribe", "one", "two"}, "subscribe requires one remote"},
}
for _, tc := range cases {
err := Run(ctx, tc.args, &bytes.Buffer{}, &bytes.Buffer{})
@ -1479,9 +1479,9 @@ func TestHelpers(t *testing.T) {
require.True(t, hybridSemanticUnavailable(store.ErrNoCompatibleEmbeddings))
require.True(t, hybridSemanticUnavailable(assertErr("semantic query embedding missing")))
require.False(t, hybridSemanticUnavailable(assertErr("other")))
opts, err := shareOptionsFromFlags("~/share", "", "")
opts, err := shareOptionsFromFlags("~/share", "git@example.com:org/archive.git", "")
require.NoError(t, err)
require.Equal(t, defaultShareRemote, opts.Remote)
require.Equal(t, "git@example.com:org/archive.git", opts.Remote)
require.Equal(t, "main", opts.Branch)
var out bytes.Buffer
require.NoError(t, printHuman(&out, syncer.SyncStats{Guilds: 1}))

13
internal/cli/share_commands.go
View File

@ -12,8 +12,6 @@ import (
"github.com/steipete/discrawl/internal/store"
)
const defaultShareRemote = "https://github.com/openclaw/discord-backup.git"
func (r *runtime) runPublish(args []string) error {
fs := flag.NewFlagSet("publish", flag.ContinueOnError)
fs.SetOutput(io.Discard)
@ -98,13 +96,10 @@ func (r *runtime) runSubscribe(args []string) error {
if err := fs.Parse(args); err != nil {
return usageErr(err)
}
remote := defaultShareRemote
if fs.NArg() > 1 {
return usageErr(errors.New("subscribe takes at most one remote"))
}
if fs.NArg() == 1 {
remote = fs.Arg(0)
if fs.NArg() != 1 {
return usageErr(errors.New("subscribe requires one remote"))
}
remote := fs.Arg(0)
cfg, err := loadConfigOrDefault(r.configPath)
if err != nil {
return err
@ -204,7 +199,7 @@ func shareOptionsFromFlags(repoPath, remote, branch string) (share.Options, erro
return share.Options{}, configErr(err)
}
if remote == "" {
remote = defaultShareRemote
return share.Options{}, configErr(errors.New("share remote is required"))
}
if branch == "" {
branch = "main"

163
internal/config/config.go
View File

@ -1,13 +1,11 @@
package config
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"sort"
"strings"
"time"
@ -35,10 +33,8 @@ type Config struct {
}
type DiscordConfig struct {
TokenSource string `toml:"token_source"`
OpenClawConfig string `toml:"openclaw_config"`
Account string `toml:"account"`
TokenEnv string `toml:"token_env"`
TokenSource string `toml:"token_source"`
TokenEnv string `toml:"token_env"`
}
type DesktopConfig struct {
@ -84,30 +80,6 @@ type TokenResolution struct {
Path string
}
type OpenClawDiscord struct {
Token string
GuildIDs []string
Path string
}
type openClawConfig struct {
Channels struct {
Discord openClawDiscord `json:"discord"`
} `json:"channels"`
Secrets openClawSecrets `json:"secrets"`
}
type openClawDiscord struct {
Token openClawSecretValue `json:"token"`
Accounts map[string]openClawDiscordAcct `json:"accounts"`
Guilds map[string]json.RawMessage `json:"guilds"`
}
type openClawDiscordAcct struct {
Token openClawSecretValue `json:"token"`
Guilds map[string]json.RawMessage `json:"guilds"`
}
func Default() Config {
home, _ := os.UserHomeDir()
base := filepath.Join(home, ".discrawl")
@ -118,10 +90,8 @@ func Default() Config {
LogDir: filepath.Join(base, "logs"),
DefaultGuildID: "",
Discord: DiscordConfig{
TokenSource: "openclaw",
OpenClawConfig: filepath.Join(home, ".openclaw", "openclaw.json"),
Account: "default",
TokenEnv: DefaultTokenEnv,
TokenSource: "env",
TokenEnv: DefaultTokenEnv,
},
Desktop: DesktopConfig{
Path: defaultDiscordDesktopPath(home),
@ -235,13 +205,7 @@ func (c *Config) Normalize() error {
}
}
if c.Discord.TokenSource == "" {
c.Discord.TokenSource = "openclaw"
}
if c.Discord.OpenClawConfig == "" {
c.Discord.OpenClawConfig = Default().Discord.OpenClawConfig
}
if c.Discord.Account == "" {
c.Discord.Account = "default"
c.Discord.TokenSource = "env"
}
if c.Discord.TokenEnv == "" {
c.Discord.TokenEnv = DefaultTokenEnv
@ -393,123 +357,6 @@ func ExpandPath(path string) (string, error) {
return filepath.Clean(os.ExpandEnv(path)), nil
}
func ResolveDiscordToken(cfg Config) (TokenResolution, error) {
if err := cfg.Normalize(); err != nil {
return TokenResolution{}, err
}
if cfg.Discord.TokenSource == "none" {
return TokenResolution{}, errors.New("discord token disabled by config")
}
if cfg.Discord.TokenSource != "env" {
openClaw, err := LoadOpenClawDiscord(cfg.Discord.OpenClawConfig, cfg.Discord.Account)
if err == nil && openClaw.Token != "" {
return TokenResolution{Token: openClaw.Token, Source: "openclaw", Path: openClaw.Path}, nil
}
if err != nil && !errors.Is(err, os.ErrNotExist) {
return TokenResolution{}, err
}
}
if envToken := NormalizeBotToken(os.Getenv(cfg.Discord.TokenEnv)); envToken != "" {
return TokenResolution{Token: envToken, Source: "env", Path: cfg.Discord.TokenEnv}, nil
}
return TokenResolution{}, errors.New("discord token not found in env or openclaw config")
}
func LoadOpenClawDiscord(path, account string) (OpenClawDiscord, error) {
paths, err := openClawCandidates(path)
if err != nil {
return OpenClawDiscord{}, err
}
for _, candidate := range paths {
info, err := loadOpenClawDiscordFile(candidate, account)
if err == nil && info.Token != "" {
return info, nil
}
if err != nil && !errors.Is(err, os.ErrNotExist) {
return OpenClawDiscord{}, err
}
}
return OpenClawDiscord{}, os.ErrNotExist
}
func loadOpenClawDiscordFile(path, account string) (OpenClawDiscord, error) {
expanded, err := ExpandPath(path)
if err != nil {
return OpenClawDiscord{}, err
}
data, err := os.ReadFile(expanded)
if err != nil {
return OpenClawDiscord{}, err
}
var payload openClawConfig
if err := json.Unmarshal(data, &payload); err != nil {
return OpenClawDiscord{}, fmt.Errorf("parse openclaw config: %w", err)
}
discord := payload.Channels.Discord
resolver := newOpenClawSecretResolver(payload.Secrets, expanded)
token, err := resolver.resolve(discord.Token)
if err != nil {
return OpenClawDiscord{}, fmt.Errorf("resolve openclaw discord token: %w", err)
}
guildIDs := mapKeys(discord.Guilds)
if token == "" {
acct := discord.Accounts[normalizeAccount(account)]
if acct.Token.empty() && account != normalizeAccount(account) {
acct = discord.Accounts[account]
}
token, err = resolver.resolve(acct.Token)
if err != nil {
return OpenClawDiscord{}, fmt.Errorf("resolve openclaw discord account token: %w", err)
}
if len(guildIDs) == 0 {
guildIDs = mapKeys(acct.Guilds)
}
}
return OpenClawDiscord{
Token: token,
GuildIDs: guildIDs,
Path: expanded,
}, nil
}
func openClawCandidates(path string) ([]string, error) {
expanded, err := ExpandPath(path)
if err != nil {
return nil, err
}
candidates := []string{expanded}
matches, err := filepath.Glob(expanded + ".bak*")
if err != nil {
return nil, err
}
sort.Strings(matches)
candidates = append(candidates, matches...)
return uniqueStrings(candidates), nil
}
func NormalizeBotToken(raw string) string {
raw = strings.TrimSpace(raw)
raw = strings.TrimPrefix(raw, "Bot ")
return strings.TrimSpace(raw)
}
func normalizeAccount(account string) string {
account = strings.TrimSpace(strings.ToLower(account))
if account == "" {
return "default"
}
return account
}
func mapKeys[V any](m map[string]V) []string {
keys := make([]string, 0, len(m))
for key := range m {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func uniqueStrings(in []string) []string {
if len(in) == 0 {
return nil

337
internal/config/config_test.go
View File

@ -15,8 +15,7 @@ func TestNormalizeFillsDefaults(t *testing.T) {
cfg := Config{}
require.NoError(t, cfg.Normalize())
require.Equal(t, 1, cfg.Version)
require.Equal(t, "openclaw", cfg.Discord.TokenSource)
require.Equal(t, "default", cfg.Discord.Account)
require.Equal(t, "env", cfg.Discord.TokenSource)
require.Equal(t, DefaultTokenEnv, cfg.Discord.TokenEnv)
require.Equal(t, defaultSyncConcurrency(), cfg.Sync.Concurrency)
require.GreaterOrEqual(t, cfg.Sync.Concurrency, 8)
@ -53,221 +52,8 @@ func TestDefaultSyncConcurrencyBounds(t *testing.T) {
require.Equal(t, 32, defaultSyncConcurrency())
}
func TestResolveDiscordTokenPrefersOpenClaw(t *testing.T) {
dir := t.TempDir()
openClawPath := filepath.Join(dir, "openclaw.json")
require.NoError(t, os.WriteFile(openClawPath, []byte(`{
"channels": {
"discord": {
"token": "Bot config-token",
"guilds": { "g1": {}, "g2": {} }
}
}
}`), 0o600))
t.Setenv(DefaultTokenEnv, "env-token")
func TestResolveDiscordTokenFromEnv(t *testing.T) {
cfg := Default()
cfg.Discord.OpenClawConfig = openClawPath
token, err := ResolveDiscordToken(cfg)
require.NoError(t, err)
require.Equal(t, "config-token", token.Token)
require.Equal(t, "openclaw", token.Source)
}
func TestResolveDiscordTokenFromOpenClawFileSecretRef(t *testing.T) {
dir := t.TempDir()
openClawPath := filepath.Join(dir, "openclaw.json")
secretsPath := filepath.Join(dir, "credentials", "secrets.json")
require.NoError(t, os.MkdirAll(filepath.Dir(secretsPath), 0o755))
require.NoError(t, os.WriteFile(secretsPath, []byte(`{
"channels": {
"discord": {
"token": "Bot file-secret-token"
}
}
}`), 0o600))
require.NoError(t, os.WriteFile(openClawPath, []byte(`{
"secrets": {
"providers": {
"filemain": {
"source": "file",
"path": "credentials/secrets.json",
"mode": "json"
}
}
},
"channels": {
"discord": {
"token": {
"source": "file",
"provider": "filemain",
"id": "/channels/discord/token"
},
"guilds": { "g1": {} }
}
}
}`), 0o600))
t.Setenv(DefaultTokenEnv, "env-token")
cfg := Default()
cfg.Discord.OpenClawConfig = openClawPath
token, err := ResolveDiscordToken(cfg)
require.NoError(t, err)
require.Equal(t, "file-secret-token", token.Token)
require.Equal(t, "openclaw", token.Source)
info, err := LoadOpenClawDiscord(openClawPath, "default")
require.NoError(t, err)
require.Equal(t, []string{"g1"}, info.GuildIDs)
}
func TestResolveDiscordAccountTokenFromOpenClawDirectFileSecretRef(t *testing.T) {
dir := t.TempDir()
openClawPath := filepath.Join(dir, "openclaw.json")
secretsPath := filepath.Join(dir, "secrets.json")
require.NoError(t, os.WriteFile(secretsPath, []byte(`{
"accounts": {
"atlas": {
"discordToken": "account-file-token"
}
}
}`), 0o600))
require.NoError(t, os.WriteFile(openClawPath, []byte(`{
"secrets": {
"filemain": {
"source": "file",
"path": "secrets.json",
"mode": "json"
}
},
"channels": {
"discord": {
"accounts": {
"atlas": {
"token": {
"source": "file",
"provider": "filemain",
"id": "/accounts/atlas/discordToken"
},
"guilds": { "g9": {} }
}
}
}
}
}`), 0o600))
info, err := LoadOpenClawDiscord(openClawPath, "atlas")
require.NoError(t, err)
require.Equal(t, "account-file-token", info.Token)
require.Equal(t, []string{"g9"}, info.GuildIDs)
}
func TestResolveDiscordTokenFromOpenClawEnvSecretRef(t *testing.T) {
dir := t.TempDir()
openClawPath := filepath.Join(dir, "openclaw.json")
require.NoError(t, os.WriteFile(openClawPath, []byte(`{
"channels": {
"discord": {
"token": {
"source": "env",
"provider": "default",
"id": "DISCRAWL_TEST_OPENCLAW_TOKEN"
}
}
}
}`), 0o600))
t.Setenv("DISCRAWL_TEST_OPENCLAW_TOKEN", "Bot env-ref-token")
info, err := LoadOpenClawDiscord(openClawPath, "default")
require.NoError(t, err)
require.Equal(t, "env-ref-token", info.Token)
}
func TestResolveDiscordTokenFromOpenClawEnvSecretRefRequiresEnvValue(t *testing.T) {
dir := t.TempDir()
openClawPath := filepath.Join(dir, "openclaw.json")
require.NoError(t, os.WriteFile(openClawPath, []byte(`{
"channels": {
"discord": {
"token": {
"source": "env",
"provider": "default",
"id": "DISCRAWL_TEST_MISSING_OPENCLAW_TOKEN"
}
}
}
}`), 0o600))
t.Setenv(DefaultTokenEnv, "fallback-token")
cfg := Default()
cfg.Discord.OpenClawConfig = openClawPath
_, err := ResolveDiscordToken(cfg)
require.ErrorContains(t, err, `environment variable "DISCRAWL_TEST_MISSING_OPENCLAW_TOKEN" is missing or empty`)
}
func TestResolveDiscordTokenFromOpenClawEnvSecretRefHonorsAllowlist(t *testing.T) {
dir := t.TempDir()
openClawPath := filepath.Join(dir, "openclaw.json")
require.NoError(t, os.WriteFile(openClawPath, []byte(`{
"secrets": {
"providers": {
"default": {
"source": "env",
"allowlist": ["OTHER_TOKEN"]
}
}
},
"channels": {
"discord": {
"token": {
"source": "env",
"provider": "default",
"id": "DISCRAWL_TEST_OPENCLAW_TOKEN"
}
}
}
}`), 0o600))
t.Setenv("DISCRAWL_TEST_OPENCLAW_TOKEN", "env-ref-token")
_, err := LoadOpenClawDiscord(openClawPath, "default")
require.ErrorContains(t, err, `environment variable "DISCRAWL_TEST_OPENCLAW_TOKEN" is not allowlisted`)
}
func TestResolveDiscordTokenFromOpenClawEnvSecretRefRejectsProviderSourceMismatch(t *testing.T) {
dir := t.TempDir()
openClawPath := filepath.Join(dir, "openclaw.json")
require.NoError(t, os.WriteFile(openClawPath, []byte(`{
"secrets": {
"providers": {
"default": {
"source": "file",
"path": "secrets.json"
}
}
},
"channels": {
"discord": {
"token": {
"source": "env",
"provider": "default",
"id": "DISCRAWL_TEST_OPENCLAW_TOKEN"
}
}
}
}`), 0o600))
t.Setenv("DISCRAWL_TEST_OPENCLAW_TOKEN", "env-ref-token")
_, err := LoadOpenClawDiscord(openClawPath, "default")
require.ErrorContains(t, err, `secret provider "default" has source "file", want env`)
}
func TestResolveDiscordTokenFallsBackToEnv(t *testing.T) {
cfg := Default()
cfg.Discord.TokenSource = "env"
cfg.Discord.OpenClawConfig = filepath.Join(t.TempDir(), "missing.json")
t.Setenv(DefaultTokenEnv, "Bot env-token")
token, err := ResolveDiscordToken(cfg)
@ -276,6 +62,25 @@ func TestResolveDiscordTokenFallsBackToEnv(t *testing.T) {
require.Equal(t, "env", token.Source)
}
func TestResolveDiscordTokenFromCustomEnv(t *testing.T) {
cfg := Default()
cfg.Discord.TokenEnv = "DISCRAWL_TEST_DISCORD_TOKEN"
t.Setenv("DISCRAWL_TEST_DISCORD_TOKEN", "custom-env-token")
token, err := ResolveDiscordToken(cfg)
require.NoError(t, err)
require.Equal(t, "custom-env-token", token.Token)
require.Equal(t, "DISCRAWL_TEST_DISCORD_TOKEN", token.Path)
}
func TestResolveDiscordTokenRequiresEnvValue(t *testing.T) {
cfg := Default()
t.Setenv(DefaultTokenEnv, "")
_, err := ResolveDiscordToken(cfg)
require.ErrorContains(t, err, `discord token not found in environment variable "DISCORD_BOT_TOKEN"`)
}
func TestResolveDiscordTokenDisabled(t *testing.T) {
cfg := Default()
cfg.Discord.TokenSource = "none"
@ -285,51 +90,12 @@ func TestResolveDiscordTokenDisabled(t *testing.T) {
require.ErrorContains(t, err, "discord token disabled")
}
func TestLoadOpenClawDiscordFromAccount(t *testing.T) {
t.Parallel()
func TestResolveDiscordTokenRejectsUnsupportedSource(t *testing.T) {
cfg := Default()
cfg.Discord.TokenSource = "legacy"
dir := t.TempDir()
openClawPath := filepath.Join(dir, "openclaw.json")
require.NoError(t, os.WriteFile(openClawPath, []byte(`{
"channels": {
"discord": {
"accounts": {
"default": {
"token": "acct-token",
"guilds": { "g3": {} }
}
}
}
}
}`), 0o600))
info, err := LoadOpenClawDiscord(openClawPath, "default")
require.NoError(t, err)
require.Equal(t, "acct-token", info.Token)
require.Equal(t, []string{"g3"}, info.GuildIDs)
}
func TestLoadOpenClawDiscordExpandsEnvToken(t *testing.T) {
dir := t.TempDir()
openClawPath := filepath.Join(dir, "openclaw.json")
t.Setenv("DISCRAWL_TEST_TOKEN", "Bot env-expanded-token")
require.NoError(t, os.WriteFile(openClawPath, []byte(`{
"channels": {
"discord": {
"accounts": {
"default": {
"token": "${DISCRAWL_TEST_TOKEN}",
"guilds": { "g3": {} }
}
}
}
}
}`), 0o600))
info, err := LoadOpenClawDiscord(openClawPath, "default")
require.NoError(t, err)
require.Equal(t, "env-expanded-token", info.Token)
require.Equal(t, []string{"g3"}, info.GuildIDs)
_, err := ResolveDiscordToken(cfg)
require.ErrorContains(t, err, `unsupported discord token_source "legacy"`)
}
func TestWriteAndLoadRoundTrip(t *testing.T) {
@ -451,7 +217,7 @@ func TestExpandPath(t *testing.T) {
require.Contains(t, path, "discrawl-test")
}
func TestResolvePathAndLoadOpenClawFallbacks(t *testing.T) {
func TestResolvePath(t *testing.T) {
dir := t.TempDir()
envPath := filepath.Join(dir, "env.toml")
t.Setenv(DefaultConfigEnv, envPath)
@ -461,43 +227,6 @@ func TestResolvePathAndLoadOpenClawFallbacks(t *testing.T) {
require.Contains(t, ResolvePath(""), filepath.Join(".discrawl", "config.toml"))
_, err := ExpandPath("")
require.ErrorContains(t, err, "empty path")
openClawPath := filepath.Join(dir, "openclaw.json")
require.NoError(t, os.WriteFile(openClawPath, []byte(`{}`), 0o600))
require.NoError(t, os.WriteFile(openClawPath+".bak", []byte(`{
"channels": {
"discord": {
"accounts": {
"Work Account": {
"token": "backup-token",
"guilds": { "g9": {} }
}
}
}
}
}`), 0o600))
info, err := LoadOpenClawDiscord(openClawPath, "Work Account")
require.NoError(t, err)
require.Equal(t, "backup-token", info.Token)
require.Equal(t, []string{"g9"}, info.GuildIDs)
_, err = LoadOpenClawDiscord(filepath.Join(dir, "missing.json"), "default")
require.ErrorIs(t, err, os.ErrNotExist)
}
func TestOpenClawCandidatesIncludesBackups(t *testing.T) {
t.Parallel()
dir := t.TempDir()
base := filepath.Join(dir, "openclaw.json")
require.NoError(t, os.WriteFile(base, []byte(`{}`), 0o600))
require.NoError(t, os.WriteFile(base+".bak", []byte(`{}`), 0o600))
require.NoError(t, os.WriteFile(base+".bak.1", []byte(`{}`), 0o600))
paths, err := openClawCandidates(base)
require.NoError(t, err)
require.Len(t, paths, 3)
}
func TestEffectiveDefaultGuildAndDirs(t *testing.T) {
@ -505,12 +234,9 @@ func TestEffectiveDefaultGuildAndDirs(t *testing.T) {
require.Equal(t, "explicit", Config{DefaultGuildID: "explicit", GuildIDs: []string{"g1"}}.EffectiveDefaultGuildID())
require.Empty(t, Config{GuildIDs: []string{"g1", "g2"}}.EffectiveDefaultGuildID())
require.Equal(t, "default", normalizeAccount(""))
require.Equal(t, "work", normalizeAccount(" Work "))
require.Equal(t, []string{"a", "b"}, uniqueStrings([]string{" a ", "", "b", "a"}))
require.Equal(t, "token", NormalizeBotToken(" token "))
require.Nil(t, uniqueStrings(nil))
require.Equal(t, []string{"a", "b"}, mapKeys(map[string]int{"b": 2, "a": 1}))
cfg := Default()
cfg.GuildIDs = []string{"g1"}
@ -542,15 +268,6 @@ func TestConfigErrorsAndBackupFallback(t *testing.T) {
require.Error(t, err)
cfg := Default()
cfg.Discord.OpenClawConfig = filepath.Join(dir, "missing.json")
_, err = ResolveDiscordToken(cfg)
require.Error(t, err)
base := filepath.Join(dir, "openclaw.json")
backup := base + ".bak"
require.NoError(t, os.WriteFile(base, []byte(`{}`), 0o600))
require.NoError(t, os.WriteFile(backup, []byte(`{"channels":{"discord":{"token":"backup-token"}}}`), 0o600))
info, err := LoadOpenClawDiscord(base, "default")
require.NoError(t, err)
require.Equal(t, "backup-token", info.Token)
}

32
internal/config/discord_token.go Normal file
View File

@ -0,0 +1,32 @@
package config
import (
"errors"
"fmt"
"os"
"strings"
)
func ResolveDiscordToken(cfg Config) (TokenResolution, error) {
if err := cfg.Normalize(); err != nil {
return TokenResolution{}, err
}
switch cfg.Discord.TokenSource {
case "none":
return TokenResolution{}, errors.New("discord token disabled by config")
case "env":
envToken := NormalizeBotToken(os.Getenv(cfg.Discord.TokenEnv))
if envToken == "" {
return TokenResolution{}, fmt.Errorf("discord token not found in environment variable %q", cfg.Discord.TokenEnv)
}
return TokenResolution{Token: envToken, Source: "env", Path: cfg.Discord.TokenEnv}, nil
default:
return TokenResolution{}, fmt.Errorf("unsupported discord token_source %q", cfg.Discord.TokenSource)
}
}
func NormalizeBotToken(raw string) string {
raw = strings.TrimSpace(raw)
raw = strings.TrimPrefix(raw, "Bot ")
return strings.TrimSpace(raw)
}

275
internal/config/openclaw_secrets.go
View File

@ -1,275 +0,0 @@
package config
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
)
type openClawSecretValue struct {
Plain string
Ref *openClawSecretRef
}
type openClawSecretRef struct {
Source string `json:"source"`
Provider string `json:"provider"`
ID string `json:"id"`
}
type openClawSecrets struct {
Providers map[string]openClawSecretProvider
Aliases map[string]openClawSecretProvider
Defaults openClawSecretDefaults
}
type openClawSecretProvider struct {
Source string `json:"source"`
Path string `json:"path"`
Mode string `json:"mode"`
Allowlist []string `json:"allowlist"`
}
type openClawSecretDefaults struct {
Env string `json:"env"`
File string `json:"file"`
Exec string `json:"exec"`
}
type openClawSecretResolver struct {
secrets openClawSecrets
configPath string
env func(string) (string, bool)
readFile func(string) ([]byte, error)
}
func (v *openClawSecretValue) UnmarshalJSON(data []byte) error {
var plain string
if err := json.Unmarshal(data, &plain); err == nil {
v.Plain = plain
v.Ref = nil
return nil
}
var ref openClawSecretRef
if err := json.Unmarshal(data, &ref); err != nil {
return err
}
v.Plain = ""
v.Ref = &ref
return nil
}
func (s *openClawSecrets) UnmarshalJSON(data []byte) error {
var aux struct {
Providers map[string]openClawSecretProvider `json:"providers"`
Defaults openClawSecretDefaults `json:"defaults"`
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
var raw map[string]json.RawMessage
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
aliases := map[string]openClawSecretProvider{}
for name, value := range raw {
if name == "providers" || name == "defaults" {
continue
}
var provider openClawSecretProvider
if err := json.Unmarshal(value, &provider); err == nil && (provider.Source != "" || provider.Path != "") {
aliases[name] = provider
}
}
s.Providers = aux.Providers
s.Aliases = aliases
s.Defaults = aux.Defaults
return nil
}
func newOpenClawSecretResolver(secrets openClawSecrets, configPath string) openClawSecretResolver {
return openClawSecretResolver{
secrets: secrets,
configPath: configPath,
env: os.LookupEnv,
readFile: os.ReadFile,
}
}
func (v openClawSecretValue) empty() bool {
return strings.TrimSpace(v.Plain) == "" && v.Ref == nil
}
func (r openClawSecretResolver) resolve(value openClawSecretValue) (string, error) {
if value.Ref == nil {
return NormalizeBotToken(os.ExpandEnv(value.Plain)), nil
}
token, err := r.resolveRef(*value.Ref)
if err != nil {
return "", err
}
if token == "" {
return "", errors.New("secret reference resolved to an empty value")
}
return token, nil
}
func (r openClawSecretResolver) resolveRef(ref openClawSecretRef) (string, error) {
source := strings.ToLower(strings.TrimSpace(ref.Source))
switch source {
case "env":
return r.resolveEnv(ref)
case "file":
return r.resolveFile(ref)
case "exec":
return "", errors.New("exec SecretRefs are not supported by discrawl init --from-openclaw")
case "":
return "", errors.New("secret ref missing source")
default:
return "", fmt.Errorf("unsupported secret ref source %q", ref.Source)
}
}
func (r openClawSecretResolver) resolveEnv(ref openClawSecretRef) (string, error) {
id := strings.TrimSpace(ref.ID)
if id == "" {
return "", errors.New("env secret ref missing id")
}
providerName := strings.TrimSpace(ref.Provider)
if providerName == "" {
providerName = defaultOpenClawSecretProvider(r.secrets.Defaults.Env, "default")
}
if provider, ok := r.secrets.provider(providerName); ok {
source := strings.ToLower(strings.TrimSpace(provider.Source))
if source != "" && source != "env" {
return "", fmt.Errorf("secret provider %q has source %q, want env", providerName, provider.Source)
}
if len(provider.Allowlist) > 0 && !stringInSlice(id, provider.Allowlist) {
return "", fmt.Errorf("environment variable %q is not allowlisted in secret provider %q", id, providerName)
}
} else if providerName != defaultOpenClawSecretProvider(r.secrets.Defaults.Env, "default") {
return "", fmt.Errorf("secret provider %q not found", providerName)
}
value, ok := r.env(id)
if !ok || strings.TrimSpace(value) == "" {
return "", fmt.Errorf("environment variable %q is missing or empty", id)
}
return NormalizeBotToken(value), nil
}
func (r openClawSecretResolver) resolveFile(ref openClawSecretRef) (string, error) {
providerName := strings.TrimSpace(ref.Provider)
if providerName == "" {
providerName = defaultOpenClawSecretProvider(r.secrets.Defaults.File, "filemain")
}
provider, ok := r.secrets.provider(providerName)
if !ok {
return "", fmt.Errorf("secret provider %q not found", providerName)
}
source := strings.ToLower(strings.TrimSpace(provider.Source))
if source != "" && source != "file" {
return "", fmt.Errorf("secret provider %q has source %q, want file", providerName, provider.Source)
}
path := strings.TrimSpace(provider.Path)
if path == "" {
return "", fmt.Errorf("secret provider %q missing path", providerName)
}
expanded, err := expandOpenClawSecretPath(path, r.configPath)
if err != nil {
return "", err
}
data, err := r.readFile(expanded)
if err != nil {
return "", err
}
mode := strings.ToLower(strings.TrimSpace(provider.Mode))
if mode == "" {
mode = "json"
}
switch mode {
case "json":
token, err := readJSONPointerString(data, ref.ID)
if err != nil {
return "", fmt.Errorf("read secret provider %q id %q: %w", providerName, ref.ID, err)
}
return NormalizeBotToken(os.ExpandEnv(token)), nil
case "singlevalue":
if strings.TrimSpace(ref.ID) != "value" {
return "", fmt.Errorf("secret provider %q singleValue mode requires id %q", providerName, "value")
}
return NormalizeBotToken(os.ExpandEnv(string(data))), nil
default:
return "", fmt.Errorf("unsupported secret provider %q mode %q", providerName, provider.Mode)
}
}
func (s openClawSecrets) provider(name string) (openClawSecretProvider, bool) {
name = strings.TrimSpace(name)
if name == "" {
name = "default"
}
if provider, ok := s.Providers[name]; ok {
return provider, true
}
provider, ok := s.Aliases[name]
return provider, ok
}
func expandOpenClawSecretPath(path, configPath string) (string, error) {
path = os.ExpandEnv(strings.TrimSpace(path))
if path == "" {
return "", errors.New("empty secret provider path")
}
if strings.HasPrefix(path, "~") || filepath.IsAbs(path) {
return ExpandPath(path)
}
return filepath.Clean(filepath.Join(filepath.Dir(configPath), path)), nil
}
func readJSONPointerString(data []byte, pointer string) (string, error) {
pointer = strings.TrimSpace(pointer)
if pointer == "" || pointer[0] != '/' {
return "", errors.New("id must be an absolute JSON pointer")
}
var current any
if err := json.Unmarshal(data, &current); err != nil {
return "", fmt.Errorf("parse secret file: %w", err)
}
for _, rawSegment := range strings.Split(pointer[1:], "/") {
segment := strings.ReplaceAll(strings.ReplaceAll(rawSegment, "~1", "/"), "~0", "~")
object, ok := current.(map[string]any)
if !ok {
return "", fmt.Errorf("segment %q is not an object", segment)
}
next, ok := object[segment]
if !ok {
return "", fmt.Errorf("segment %q not found", segment)
}
current = next
}
secret, ok := current.(string)
if !ok {
return "", errors.New("secret value is not a string")
}
return secret, nil
}
func defaultOpenClawSecretProvider(configured, fallback string) string {
configured = strings.TrimSpace(configured)
if configured != "" {
return configured
}
return fallback
}
func stringInSlice(needle string, haystack []string) bool {
for _, item := range haystack {
if item == needle {
return true
}
}
return false
}

138
internal/config/openclaw_secrets_test.go
View File

@ -1,138 +0,0 @@
package config
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
func TestOpenClawSecretResolverUsesFileDefaultProvider(t *testing.T) {
dir := t.TempDir()
secretsPath := filepath.Join(dir, "secrets.json")
require.NoError(t, writeTestFile(secretsPath, `{
"discord": {
"token": "Bot default-file-token"
}
}`))
resolver := newOpenClawSecretResolver(openClawSecrets{
Defaults: openClawSecretDefaults{File: "vaultfile"},
Providers: map[string]openClawSecretProvider{
"vaultfile": {
Source: "file",
Path: "secrets.json",
Mode: "json",
},
},
}, filepath.Join(dir, "openclaw.json"))
token, err := resolver.resolve(openClawSecretValue{Ref: &openClawSecretRef{
Source: "file",
ID: "/discord/token",
}})
require.NoError(t, err)
require.Equal(t, "default-file-token", token)
}
func TestOpenClawSecretResolverReadsSingleValueFile(t *testing.T) {
dir := t.TempDir()
require.NoError(t, writeTestFile(filepath.Join(dir, "discord-token.txt"), "Bot single-value-token\n"))
resolver := newOpenClawSecretResolver(openClawSecrets{
Providers: map[string]openClawSecretProvider{
"filemain": {
Source: "file",
Path: "discord-token.txt",
Mode: "singleValue",
},
},
}, filepath.Join(dir, "openclaw.json"))
token, err := resolver.resolve(openClawSecretValue{Ref: &openClawSecretRef{
Source: "file",
Provider: "filemain",
ID: "value",
}})
require.NoError(t, err)
require.Equal(t, "single-value-token", token)
}
func TestOpenClawSecretResolverRejectsSingleValueFilePointerIDs(t *testing.T) {
dir := t.TempDir()
require.NoError(t, writeTestFile(filepath.Join(dir, "discord-token.txt"), "token"))
resolver := newOpenClawSecretResolver(openClawSecrets{
Providers: map[string]openClawSecretProvider{
"filemain": {
Source: "file",
Path: "discord-token.txt",
Mode: "singleValue",
},
},
}, filepath.Join(dir, "openclaw.json"))
_, err := resolver.resolve(openClawSecretValue{Ref: &openClawSecretRef{
Source: "file",
Provider: "filemain",
ID: "/discord/token",
}})
require.ErrorContains(t, err, `singleValue mode requires id "value"`)
}
func TestOpenClawSecretResolverRejectsEmptyFileSecret(t *testing.T) {
dir := t.TempDir()
require.NoError(t, writeTestFile(filepath.Join(dir, "secrets.json"), `{"discord":{"token":" "}}`))
resolver := newOpenClawSecretResolver(openClawSecrets{
Providers: map[string]openClawSecretProvider{
"filemain": {
Source: "file",
Path: "secrets.json",
Mode: "json",
},
},
}, filepath.Join(dir, "openclaw.json"))
_, err := resolver.resolve(openClawSecretValue{Ref: &openClawSecretRef{
Source: "file",
Provider: "filemain",
ID: "/discord/token",
}})
require.ErrorContains(t, err, "secret reference resolved to an empty value")
}
func TestOpenClawSecretResolverRejectsFileProviderSourceMismatch(t *testing.T) {
dir := t.TempDir()
resolver := newOpenClawSecretResolver(openClawSecrets{
Providers: map[string]openClawSecretProvider{
"filemain": {
Source: "env",
Path: "secrets.json",
},
},
}, filepath.Join(dir, "openclaw.json"))
_, err := resolver.resolve(openClawSecretValue{Ref: &openClawSecretRef{
Source: "file",
Provider: "filemain",
ID: "/discord/token",
}})
require.ErrorContains(t, err, `secret provider "filemain" has source "env", want file`)
}
func TestOpenClawSecretResolverRejectsExecRefsClearly(t *testing.T) {
resolver := newOpenClawSecretResolver(openClawSecrets{}, filepath.Join(t.TempDir(), "openclaw.json"))
_, err := resolver.resolve(openClawSecretValue{Ref: &openClawSecretRef{
Source: "exec",
Provider: "vault",
ID: "discord/token",
}})
require.ErrorContains(t, err, "exec SecretRefs are not supported by discrawl init --from-openclaw")
}
func writeTestFile(path, contents string) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, []byte(contents), 0o600)
}

10
internal/store/store_test.go
View File

@ -410,8 +410,8 @@ func TestSearchMessagesPrefersRecentMessageIDs(t *testing.T) {
AuthorName: "Peter",
MessageType: 0,
CreatedAt: base.Format(time.RFC3339Nano),
Content: "OpenClaw first hit",
NormalizedContent: "openclaw first hit",
Content: "Discord first hit",
NormalizedContent: "discord first hit",
RawJSON: `{}`,
}))
require.NoError(t, s.UpsertMessage(ctx, MessageRecord{
@ -423,12 +423,12 @@ func TestSearchMessagesPrefersRecentMessageIDs(t *testing.T) {
AuthorName: "Peter",
MessageType: 0,
CreatedAt: base.Add(time.Minute).Format(time.RFC3339Nano),
Content: "OpenClaw newest hit",
NormalizedContent: "openclaw newest hit",
Content: "Discord newest hit",
NormalizedContent: "discord newest hit",
RawJSON: `{}`,
}))
results, err := s.SearchMessages(ctx, SearchOptions{Query: "OpenClaw", Limit: 1})
results, err := s.SearchMessages(ctx, SearchOptions{Query: "Discord", Limit: 1})
require.NoError(t, err)
require.Len(t, results, 1)
require.Equal(t, "1489845247147118682", results[0].MessageID)

422
scripts/discord-backup-field-notes.sh
View File

@ -1,422 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
if [ "$#" -ne 2 ]; then
echo "usage: $0 <discrawl-config> <backup-repo>" >&2
exit 2
fi
CONFIG=$1
BACKUP_REPO=$2
OPENCLAW_BIN=${OPENCLAW_BIN:-openclaw}
OPENCLAW_TIMEOUT=${OPENCLAW_TIMEOUT:-300}
OPENCLAW_THINKING=${OPENCLAW_THINKING:-low}
GH_REPO=${DISCORD_FIELD_NOTES_GITHUB_REPO:-openclaw/openclaw}
START_MARKER="<!-- discrawl-field-notes:start -->"
END_MARKER="<!-- discrawl-field-notes:end -->"
TMP_DIR=$(mktemp -d)
trap 'rm -rf "$TMP_DIR"' EXIT
date_utc_days_ago() {
local days=$1
if date -u -d "$days days ago" '+%Y-%m-%d' >/dev/null 2>&1; then
date -u -d "$days days ago" '+%Y-%m-%d'
else
date -u -v-"$days"d '+%Y-%m-%d'
fi
}
run_sql() {
local title=$1
local query=$2
local output
{
printf "\n## %s\n\n" "$title"
if output=$(DISCRAWL_NO_AUTO_UPDATE=1 go run ./cmd/discrawl --config "$CONFIG" --json sql "$query" 2>&1); then
printf '%s\n' "$output" | jq -c .
else
printf '[]\n'
printf '\n_query skipped: %s_\n' "$(printf '%s' "$output" | tail -n 1)"
fi
} >>"$TMP_DIR/context.md"
}
fallback_query() {
local query=$1
if ! DISCRAWL_NO_AUTO_UPDATE=1 go run ./cmd/discrawl --config "$CONFIG" --json sql "$query"; then
printf '[]\n'
fi
}
write_fallback_notes() {
local generated_at
local latest_message
generated_at=$(date -u '+%Y-%m-%d %H:%M UTC')
latest_message=$(
fallback_query "select max(created_at) as latest_message from messages;" |
jq -r 'if type == "array" then .[0].latest_message // "unknown" else .rows[0][0] // "unknown" end'
)
fallback_query "$recent_human_cte
select topic, count(*) as matches
from (
select case
when lower(body) like '%thank%' or lower(body) like '%helpful%' or lower(body) like '%useful%' then 'Helpful answers and practical fixes'
when lower(body) like '%fast%' or lower(body) like '%speed%' or lower(body) like '%quick%' then 'Speed and responsiveness'
when lower(body) like '%agent%' or lower(body) like '%workflow%' or lower(body) like '%automatic%' then 'Agent workflows and automation'
when lower(body) like '%skill%' or lower(body) like '%tool%' or lower(body) like '%mcp%' then 'Skills, tools, and MCP integrations'
else 'General positive feedback'
end as topic
from recent
where $body_love_terms
)
group by 1
order by matches desc
limit 4;
" | jq -r '
if type == "array" then .[] else (.rows // [])[] | {topic: .[0], matches: .[1]} end |
"- " + .topic + ": " + (.matches | tostring) + " positive mentions in the recent sample."
' >"$TMP_DIR/fallback-love.md"
fallback_query "$recent_human_cte
select topic, count(*) as matches
from (
select case
when lower(body) like '%overload%' or lower(body) like '%fallback%' or lower(body) like '%provider%' or lower(body) like '%model%' then 'Provider reliability and model fallback'
when lower(body) like '%token%' or lower(body) like '%secret%' or lower(body) like '%auth%' or lower(body) like '%config%' or lower(body) like '%install%' then 'Setup, auth, and configuration'
when lower(body) like '%github%' or lower(body) like '%repo%' or lower(body) like '%pr%' or lower(body) like '%issue%' then 'GitHub and repository workflow'
when lower(body) like '%skill%' or lower(body) like '%tool%' or lower(body) like '%mcp%' then 'Skills, tools, and runtime bridges'
when lower(body) like '%encoding%' or lower(body) like '%character%' or lower(body) like '%message%' then 'Message quality, editing, and encoding'
else 'General bugs, failures, and confusing behavior'
end as topic
from recent
where $body_complaint_terms
)
group by 1
order by matches desc
limit 4;
" | jq -r '
if type == "array" then .[] else (.rows // [])[] | {topic: .[0], matches: .[1]} end |
"- " + .topic + ": " + (.matches | tostring) + " complaint-flavored mentions in the recent sample; compare with the issue/PR cluster below."
' >"$TMP_DIR/fallback-complaints.md"
if command -v gh >/dev/null 2>&1; then
gh search issues --repo "$GH_REPO" --updated ">=$github_since" \
--json number,title,state,updatedAt,url,labels \
--limit 8 | jq -r '.[0:3][]? | "- Issue #" + (.number | tostring) + " (" + .state + "): [" + .title + "](" + .url + ")"' >"$TMP_DIR/fallback-issues.md" || :
gh search prs --repo "$GH_REPO" --updated ">=$github_since" \
--json number,title,state,updatedAt,url,labels \
--limit 25 | jq -r '
def score($title):
[ "fix", "bug", "fail", "error", "provider", "fallback", "auth", "config", "skill", "tool", "github", "encoding" ]
| map(if $title | contains(.) then 1 else 0 end)
| add;
map(. + {score: score(.title | ascii_downcase)})
| map(select(.state == "open")) as $open
| ($open // .) | sort_by(.score, .updatedAt) | reverse | .[0] // empty
| "- PR #" + (.number | tostring) + ": [" + .title + "](" + .url + ") is the best recent watch candidate from title/recency signals."
' >"$TMP_DIR/fallback-pr.md" || :
fi
if [ ! -s "$TMP_DIR/fallback-love.md" ]; then
printf '%s\n' "- Not enough clear positive signal in the latest 30-day window." >"$TMP_DIR/fallback-love.md"
fi
if [ ! -s "$TMP_DIR/fallback-complaints.md" ]; then
printf '%s\n' "- Not enough clear complaint signal in the latest 30-day window." >"$TMP_DIR/fallback-complaints.md"
fi
if [ -s "$TMP_DIR/fallback-issues.md" ]; then
{
printf '\n'
printf '%s\n' "Related GitHub issue cluster:"
cat "$TMP_DIR/fallback-issues.md"
} >>"$TMP_DIR/fallback-complaints.md"
fi
if [ ! -s "$TMP_DIR/fallback-pr.md" ]; then
printf '%s\n' "- No recent PR signal available from GitHub search." >"$TMP_DIR/fallback-pr.md"
fi
{
printf '### Field Notes\n\n'
printf 'Last generated: %s\n\n' "$generated_at"
printf '_OpenClaw agent timed out; generated deterministic field notes from the same archive and GitHub context._\n\n'
printf 'Latest archived message: %s\n\n' "$latest_message"
printf '#### What People Love\n'
cat "$TMP_DIR/fallback-love.md"
printf '\n#### What People Complain About\n'
cat "$TMP_DIR/fallback-complaints.md"
printf '\n#### Best PR To Watch\n'
cat "$TMP_DIR/fallback-pr.md"
}
}
run_openclaw_agent() {
local -a cmd=(
"$OPENCLAW_BIN" agent
--local
--agent main
--thinking "$OPENCLAW_THINKING"
--timeout "$OPENCLAW_TIMEOUT"
--json
--message "$(cat "$TMP_DIR/prompt.md")"
)
if command -v timeout >/dev/null 2>&1; then
timeout "${OPENCLAW_TIMEOUT}s" "${cmd[@]}"
elif command -v gtimeout >/dev/null 2>&1; then
gtimeout "${OPENCLAW_TIMEOUT}s" "${cmd[@]}"
else
"${cmd[@]}"
fi
}
extract_openclaw_text() {
if jq -r '
[
.payloads[]?.text?,
.finalAssistantVisibleText?,
.finalAssistantRawText?,
.result.finalAssistantVisibleText?,
.result.finalAssistantRawText?,
(.. | objects | .finalAssistantVisibleText?),
(.. | objects | .finalAssistantRawText?)
]
| map(select(type == "string" and length > 0))[0] // empty
' "$TMP_DIR/openclaw-result.json" 2>/dev/null; then
return
fi
jq -Rrs '
def field_notes:
[
.payloads[]?.text?,
.finalAssistantVisibleText?,
.finalAssistantRawText?,
.result.finalAssistantVisibleText?,
.result.finalAssistantRawText?,
(.. | objects | .finalAssistantVisibleText?),
(.. | objects | .finalAssistantRawText?)
]
| map(select(type == "string" and length > 0))[0] // empty;
capture("(?s)(?<json>\\{.*\\})").json
| fromjson
| field_notes
' "$TMP_DIR/openclaw-result.json" 2>/dev/null || true
}
anchor_expr="(select max(created_at) from messages)"
since_7="strftime('%Y-%m-%dT%H:%M:%fZ', datetime($anchor_expr, '-7 days'))"
since_30="strftime('%Y-%m-%dT%H:%M:%fZ', datetime($anchor_expr, '-30 days'))"
human_filter="lower(coalesce(mem.username, mem.display_name, m.author_id, '')) not in ('github', 'dependabot')"
love_terms="(lower(coalesce(normalized_content, content, '')) like '%love%' or lower(coalesce(normalized_content, content, '')) like '%great%' or lower(coalesce(normalized_content, content, '')) like '%awesome%' or lower(coalesce(normalized_content, content, '')) like '%amazing%' or lower(coalesce(normalized_content, content, '')) like '%thanks%' or lower(coalesce(normalized_content, content, '')) like '%thank you%' or lower(coalesce(normalized_content, content, '')) like '%works%' or lower(coalesce(normalized_content, content, '')) like '%useful%' or lower(coalesce(normalized_content, content, '')) like '%helpful%' or lower(coalesce(normalized_content, content, '')) like '%fast%')"
complaint_terms="(lower(coalesce(normalized_content, content, '')) like '%bug%' or lower(coalesce(normalized_content, content, '')) like '%broken%' or lower(coalesce(normalized_content, content, '')) like '%fail%' or lower(coalesce(normalized_content, content, '')) like '%error%' or lower(coalesce(normalized_content, content, '')) like '%crash%' or lower(coalesce(normalized_content, content, '')) like '%regression%' or lower(coalesce(normalized_content, content, '')) like '%slow%' or lower(coalesce(normalized_content, content, '')) like '%confusing%' or lower(coalesce(normalized_content, content, '')) like '%annoying%' or lower(coalesce(normalized_content, content, '')) like '%not working%' or lower(coalesce(normalized_content, content, '')) like '%cannot%' or lower(coalesce(normalized_content, content, '')) like '%can''t%')"
body_love_terms="(lower(body) like '%love%' or lower(body) like '%great%' or lower(body) like '%awesome%' or lower(body) like '%amazing%' or lower(body) like '%thanks%' or lower(body) like '%thank you%' or lower(body) like '%works%' or lower(body) like '%useful%' or lower(body) like '%helpful%' or lower(body) like '%fast%')"
body_complaint_terms="(lower(body) like '%bug%' or lower(body) like '%broken%' or lower(body) like '%fail%' or lower(body) like '%error%' or lower(body) like '%crash%' or lower(body) like '%regression%' or lower(body) like '%slow%' or lower(body) like '%confusing%' or lower(body) like '%annoying%' or lower(body) like '%not working%' or lower(body) like '%cannot%' or lower(body) like '%can''t%')"
recent_human_cte="
with recent as (
select
m.created_at,
coalesce(nullif(c.name, ''), m.channel_id) as channel,
coalesce(nullif(mem.display_name, ''), nullif(mem.username, ''), m.author_id, '') as author,
coalesce(nullif(m.content, ''), m.normalized_content, '') as body
from messages m
left join channels c on c.id = m.channel_id
left join members mem on mem.guild_id = m.guild_id and mem.user_id = m.author_id
where $human_filter
order by m.rowid desc
limit 50000
)"
github_since=$(date_utc_days_ago 30)
cat >"$TMP_DIR/context.md" <<EOF
# Discord Backup Field Notes Context
Generated at: $(date -u '+%Y-%m-%d %H:%M UTC')
GitHub repo for issue/PR correlation: $GH_REPO
Rules for interpreting this context:
- Ignore bot/integration message volume as an insight.
- Do not treat "GitHub posted the most" as funny or useful.
- Prefer human discussion samples and GitHub issue/PR titles when explaining trends.
- Avoid repeating bracketed channel prefixes such as [openclaw]; use concise channel names.
EOF
run_sql "Archive Totals" "
select
count(*) as messages,
count(distinct channel_id) as channels,
count(distinct author_id) as authors,
max(created_at) as latest_message
from messages;
"
run_sql "Activity Windows" "
select '24h' as window, count(*) as messages, count(distinct author_id) as authors, count(distinct channel_id) as channels
from messages where created_at >= strftime('%Y-%m-%dT%H:%M:%fZ', datetime($anchor_expr, '-1 day'))
union all
select '7d', count(*), count(distinct author_id), count(distinct channel_id)
from messages where created_at >= $since_7
union all
select '30d', count(*), count(distinct author_id), count(distinct channel_id)
from messages where created_at >= $since_30;
"
run_sql "Human Hot Channels This Week" "
$recent_human_cte
select channel, count(*) as messages
from recent
group by 1
order by messages desc
limit 8;
"
run_sql "What People Seem To Love In Recent Messages" "
$recent_human_cte
select channel, count(*) as matches
from recent
where $body_love_terms
group by 1
order by matches desc
limit 8;
"
run_sql "Love Samples" "
${recent_human_cte}
select created_at, channel, author, substr(body, 1, 260) as sample
from recent
where $body_love_terms
order by created_at desc
limit 10;
"
run_sql "What People Complain About In Recent Messages" "
$recent_human_cte
select channel, count(*) as matches
from recent
where $body_complaint_terms
group by 1
order by matches desc
limit 8;
"
run_sql "Complaint Samples" "
${recent_human_cte}
select created_at, channel, author, substr(body, 1, 320) as sample
from recent
where $body_complaint_terms
order by created_at desc
limit 12;
"
{
printf "\n## GitHub Pull Requests Updated This Month\n\n"
if command -v gh >/dev/null 2>&1; then
gh search prs --repo "$GH_REPO" --updated ">=$github_since" \
--json number,title,state,updatedAt,url,labels \
--limit 25 | jq -c . || printf "[]\n"
else
printf "gh unavailable\n"
fi
printf "\n## GitHub Issues Updated This Month\n\n"
if command -v gh >/dev/null 2>&1; then
gh search issues --repo "$GH_REPO" --updated ">=$github_since" \
--json number,title,state,updatedAt,url,labels \
--limit 25 | jq -c . || printf "[]\n"
else
printf "gh unavailable\n"
fi
} >>"$TMP_DIR/context.md"
cat >"$TMP_DIR/prompt.md" <<EOF
You are an OpenClaw agent writing the private Discord backup field notes.
Use the context below. Return ONLY Markdown for insertion between README markers.
Required shape:
### Field Notes
Last generated: <UTC timestamp>
#### What People Love
- 3-5 specific bullets.
#### What People Complain About
- 3-5 specific bullets.
- Correlate complaint topics with GitHub issue/PR clusters when the context supports it.
#### Best PR To Watch
- Pick the likely highest-leverage PR from GitHub context, include its title and URL if available, and explain why.
Hard rules:
- Do not say GitHub being the top poster is funny or noteworthy.
- Do not overuse the literal "[openclaw]" prefix.
- Prefer concrete product/topic names over channel-volume trivia.
- No secrets, tokens, raw internal IDs, or private URLs.
- No bullying or dunking on individual people.
- Be useful first, funny only if the evidence earns it.
Context:
$(cat "$TMP_DIR/context.md")
EOF
if run_openclaw_agent >"$TMP_DIR/openclaw-result.json" 2>&1; then
extract_openclaw_text >"$TMP_DIR/field-notes.md"
else
echo "openclaw field notes failed; using deterministic fallback" >&2
write_fallback_notes >"$TMP_DIR/field-notes.md"
fi
if [ ! -s "$TMP_DIR/field-notes.md" ]; then
echo "openclaw did not return field notes text; using deterministic fallback" >&2
write_fallback_notes >"$TMP_DIR/field-notes.md"
fi
awk -v start="$START_MARKER" -v end="$END_MARKER" -v notes="$TMP_DIR/field-notes.md" '
BEGIN {
while ((getline line < notes) > 0) {
body = body line "\n"
}
close(notes)
}
index($0, start) {
if (!wrote_notes) {
print start
printf "%s", body
emit_end = 1
wrote_notes = 1
}
in_notes = 1
next
}
index($0, end) {
if (emit_end) {
print end
emit_end = 0
}
in_notes = 0
next
}
in_notes {
next
}
{
print
if (!wrote_notes && index($0, "<!-- discrawl-report:end -->")) {
print ""
print start
printf "%s", body
print end
wrote_notes = 1
}
}
END {
if (!wrote_notes) {
print ""
print start
printf "%s", body
print end
}
}
' "$BACKUP_REPO/README.md" >"$TMP_DIR/README.md"
mv "$TMP_DIR/README.md" "$BACKUP_REPO/README.md"

311
scripts/field_notes_script_test.go
View File

@ -1,311 +0,0 @@
package scripts_test
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestDiscordBackupFieldNotesScriptUsesOpenClawAndReplacesReadmeBlock(t *testing.T) {
root, err := filepath.Abs("..")
require.NoError(t, err)
tmp := t.TempDir()
binDir := filepath.Join(tmp, "bin")
require.NoError(t, os.Mkdir(binDir, 0o755))
openclawCalls := filepath.Join(tmp, "openclaw-calls.txt")
writeExecutable(t, filepath.Join(binDir, "go"), `#!/usr/bin/env bash
printf '[{"ok":true}]'
`)
writeExecutable(t, filepath.Join(binDir, "jq"), `#!/usr/bin/env bash
if [ "${1:-}" = "-r" ]; then
cat <<'MD'
### Field Notes
Last generated: 2026-04-21 07:00 UTC
#### What People Love
- Fast Git sync.
#### What People Complain About
- Query pain maps to PR #42.
#### Best PR To Watch
- PR #42 because it closes the noisy issue cluster.
MD
else
cat
fi
`)
writeExecutable(t, filepath.Join(binDir, "gh"), `#!/usr/bin/env bash
printf '[{"number":42,"title":"Improve query sync","state":"open","url":"https://github.com/openclaw/openclaw/pull/42"}]'
`)
writeExecutable(t, filepath.Join(binDir, "openclaw"), `#!/usr/bin/env bash
printf '%s\n' "$*" > "$OPENCLAW_CALLS"
printf '{"payloads":[{"text":"unused"}]}'
`)
backupRepo := filepath.Join(tmp, "backup")
require.NoError(t, os.Mkdir(backupRepo, 0o755))
readmePath := filepath.Join(backupRepo, "README.md")
require.NoError(t, os.WriteFile(readmePath, []byte(`# Discord Backup
<!-- discrawl-report:start -->
## Discord Activity Report
<!-- discrawl-report:end -->
<!-- discrawl-field-notes:start -->
old notes
<!-- discrawl-field-notes:end -->
`), 0o644))
cmd := exec.Command("bash", "scripts/discord-backup-field-notes.sh", filepath.Join(tmp, "config.toml"), backupRepo)
cmd.Dir = root
cmd.Env = append(os.Environ(),
"PATH="+binDir+string(os.PathListSeparator)+os.Getenv("PATH"),
"OPENCLAW_BIN="+filepath.Join(binDir, "openclaw"),
"OPENCLAW_CALLS="+openclawCalls,
"GH_TOKEN=test-token",
)
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
readme, err := os.ReadFile(readmePath)
require.NoError(t, err)
text := string(readme)
require.Contains(t, text, "#### What People Love")
require.Contains(t, text, "#### What People Complain About")
require.Contains(t, text, "#### Best PR To Watch")
require.Contains(t, text, "PR #42")
require.NotContains(t, text, "old notes")
require.Equal(t, 1, strings.Count(text, "<!-- discrawl-field-notes:start -->"))
require.Equal(t, 1, strings.Count(text, "<!-- discrawl-field-notes:end -->"))
calls, err := os.ReadFile(openclawCalls)
require.NoError(t, err)
require.Contains(t, string(calls), "agent --local")
require.Contains(t, string(calls), "--thinking low")
require.NotContains(t, text, "GitHub posted the most")
}
func TestDiscordBackupFieldNotesScriptDedupesExistingReadmeBlocks(t *testing.T) {
root, err := filepath.Abs("..")
require.NoError(t, err)
tmp := t.TempDir()
binDir := filepath.Join(tmp, "bin")
require.NoError(t, os.Mkdir(binDir, 0o755))
writeExecutable(t, filepath.Join(binDir, "go"), `#!/usr/bin/env bash
printf '[{"ok":true}]'
`)
writeExecutable(t, filepath.Join(binDir, "jq"), `#!/usr/bin/env bash
if [ "${1:-}" = "-r" ]; then
cat <<'MD'
### Field Notes
Last generated: 2026-04-21 10:00 UTC
#### What People Love
- Fresh notes.
#### What People Complain About
- Duplicate marker blocks.
#### Best PR To Watch
- PR #100 because it keeps README clean.
MD
else
cat
fi
`)
writeExecutable(t, filepath.Join(binDir, "gh"), `#!/usr/bin/env bash
printf '[]'
`)
writeExecutable(t, filepath.Join(binDir, "openclaw"), `#!/usr/bin/env bash
printf '{"payloads":[{"text":"unused"}]}'
`)
backupRepo := filepath.Join(tmp, "backup")
require.NoError(t, os.Mkdir(backupRepo, 0o755))
readmePath := filepath.Join(backupRepo, "README.md")
require.NoError(t, os.WriteFile(readmePath, []byte(`# Discord Backup
<!-- discrawl-report:start -->
## Discord Activity Report
<!-- discrawl-report:end -->
<!-- discrawl-field-notes:start -->
old notes one
<!-- discrawl-field-notes:end -->
<!-- discrawl-field-notes:start -->
old notes two
<!-- discrawl-field-notes:end -->
`), 0o644))
cmd := exec.Command("bash", "scripts/discord-backup-field-notes.sh", filepath.Join(tmp, "config.toml"), backupRepo)
cmd.Dir = root
cmd.Env = append(os.Environ(),
"PATH="+binDir+string(os.PathListSeparator)+os.Getenv("PATH"),
"OPENCLAW_BIN="+filepath.Join(binDir, "openclaw"),
"GH_TOKEN=test-token",
)
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
readme, err := os.ReadFile(readmePath)
require.NoError(t, err)
text := string(readme)
require.Contains(t, text, "Fresh notes")
require.NotContains(t, text, "old notes one")
require.NotContains(t, text, "old notes two")
require.Equal(t, 1, strings.Count(text, "<!-- discrawl-field-notes:start -->"))
require.Equal(t, 1, strings.Count(text, "<!-- discrawl-field-notes:end -->"))
}
func TestDiscordBackupFieldNotesScriptParsesOpenClawJSONFromStderr(t *testing.T) {
root, err := filepath.Abs("..")
require.NoError(t, err)
tmp := t.TempDir()
binDir := filepath.Join(tmp, "bin")
require.NoError(t, os.Mkdir(binDir, 0o755))
writeExecutable(t, filepath.Join(binDir, "go"), `#!/usr/bin/env bash
printf '[{"ok":true}]'
`)
writeExecutable(t, filepath.Join(binDir, "jq"), `#!/usr/bin/env bash
case "$*" in
*payloads*|*field_notes*)
cat <<'MD'
### Field Notes
Last generated: 2026-04-21 09:45 UTC
#### What People Love
- Useful debugging.
#### What People Complain About
- Runtime failures correlate with issue clusters.
#### Best PR To Watch
- PR #99 because it fixes the cluster.
MD
;;
*)
cat
;;
esac
`)
writeExecutable(t, filepath.Join(binDir, "gh"), `#!/usr/bin/env bash
printf '[]'
`)
writeExecutable(t, filepath.Join(binDir, "openclaw"), `#!/usr/bin/env bash
cat >&2 <<'JSON'
{"payloads":[{"text":"### Field Notes\n\nLast generated: 2026-04-21 09:45 UTC\n\n#### What People Love\n- Useful debugging.\n\n#### What People Complain About\n- Runtime failures correlate with issue clusters.\n\n#### Best PR To Watch\n- PR #99 because it fixes the cluster."}],"finalAssistantVisibleText":"### Field Notes\n\nLast generated: 2026-04-21 09:45 UTC\n\n#### What People Love\n- Useful debugging.\n\n#### What People Complain About\n- Runtime failures correlate with issue clusters.\n\n#### Best PR To Watch\n- PR #99 because it fixes the cluster."}
JSON
`)
backupRepo := filepath.Join(tmp, "backup")
require.NoError(t, os.Mkdir(backupRepo, 0o755))
readmePath := filepath.Join(backupRepo, "README.md")
require.NoError(t, os.WriteFile(readmePath, []byte(`# Discord Backup
<!-- discrawl-report:start -->
## Discord Activity Report
<!-- discrawl-report:end -->
`), 0o644))
cmd := exec.Command("bash", "scripts/discord-backup-field-notes.sh", filepath.Join(tmp, "config.toml"), backupRepo)
cmd.Dir = root
cmd.Env = append(os.Environ(),
"PATH="+binDir+string(os.PathListSeparator)+os.Getenv("PATH"),
"OPENCLAW_BIN="+filepath.Join(binDir, "openclaw"),
"GH_TOKEN=test-token",
)
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
require.NotContains(t, string(out), "using deterministic fallback")
readme, err := os.ReadFile(readmePath)
require.NoError(t, err)
text := string(readme)
require.Contains(t, text, "Useful debugging")
require.Contains(t, text, "PR #99")
require.NotContains(t, text, "OpenClaw agent timed out")
}
func TestDiscordBackupFieldNotesScriptFallsBackWhenOpenClawFails(t *testing.T) {
root, err := filepath.Abs("..")
require.NoError(t, err)
tmp := t.TempDir()
binDir := filepath.Join(tmp, "bin")
require.NoError(t, os.Mkdir(binDir, 0o755))
writeExecutable(t, filepath.Join(binDir, "go"), `#!/usr/bin/env bash
printf '[{"latest_message":"2026-04-21T07:10:46Z","channel":"general","matches":3}]'
`)
writeExecutable(t, filepath.Join(binDir, "jq"), `#!/usr/bin/env bash
if [ "${1:-}" = "-c" ]; then
cat
exit 0
fi
if [ "${1:-}" = "-r" ]; then
expr="${2:-}"
cat >/dev/null
case "$expr" in
*latest_message*) printf '2026-04-21T07:10:46Z\n' ;;
*positive*) printf -- '- general: 3 positive mentions in the last 30 days.\n' ;;
*Issue*) printf -- '- Issue #7 (open): [Fix query pain](https://github.com/openclaw/openclaw/issues/7)\n' ;;
*PR*) printf -- '- PR #9: [Improve sync](https://github.com/openclaw/openclaw/pull/9) looks like the highest-leverage recent PR because it is active in the same window as the complaint cluster.\n' ;;
*complaint*) printf -- '- bugs: 2 complaint-flavored mentions in the last 30 days; compare this with the issue/PR cluster below.\n' ;;
*) cat ;;
esac
fi
`)
writeExecutable(t, filepath.Join(binDir, "gh"), `#!/usr/bin/env bash
printf '[{"number":9,"title":"Improve sync","state":"open","url":"https://github.com/openclaw/openclaw/pull/9"}]'
`)
writeExecutable(t, filepath.Join(binDir, "openclaw"), `#!/usr/bin/env bash
echo "context deadline exceeded" >&2
exit 1
`)
backupRepo := filepath.Join(tmp, "backup")
require.NoError(t, os.Mkdir(backupRepo, 0o755))
readmePath := filepath.Join(backupRepo, "README.md")
require.NoError(t, os.WriteFile(readmePath, []byte(`# Discord Backup
<!-- discrawl-report:start -->
## Discord Activity Report
<!-- discrawl-report:end -->
`), 0o644))
cmd := exec.Command("bash", "scripts/discord-backup-field-notes.sh", filepath.Join(tmp, "config.toml"), backupRepo)
cmd.Dir = root
cmd.Env = append(os.Environ(),
"PATH="+binDir+string(os.PathListSeparator)+os.Getenv("PATH"),
"OPENCLAW_BIN="+filepath.Join(binDir, "openclaw"),
"GH_TOKEN=test-token",
)
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
require.Contains(t, string(out), "openclaw field notes failed")
readme, err := os.ReadFile(readmePath)
require.NoError(t, err)
text := string(readme)
require.Contains(t, text, "OpenClaw agent timed out")
require.Contains(t, text, "#### What People Love")
require.Contains(t, text, "#### What People Complain About")
require.Contains(t, text, "Issue #7")
require.Contains(t, text, "PR #9")
require.NotContains(t, text, "GitHub posted the most")
}
func writeExecutable(t *testing.T, path string, body string) {
t.Helper()
require.NoError(t, os.WriteFile(path, []byte(body), 0o755))
}