fix: refresh portable store before reads

This commit is contained in:
Peter Steinberger 2026-04-27 23:55:06 +01:00
parent d35845f959
commit 69ae81e366
No known key found for this signature in database
3 changed files with 131 additions and 0 deletions

View File

@ -4,3 +4,4 @@
- Add `gitcrawl sync --state open|closed|all` so incremental backups can refresh recently closed issues and pull requests.
- Let `gitcrawl search` fall back to compact thread title/body data when portable stores have pruned generated document indexes.
- Refresh clean portable-store checkouts before read-only commands so `search`, `threads`, clusters, and the TUI see freshly published GitHub backup data automatically.

View File

@ -3,8 +3,10 @@ package cli
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
@ -132,6 +134,96 @@ func TestSyncPortableStoreResetsDirtyCache(t *testing.T) {
}
}
func TestReadCommandRefreshesPortableStore(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
remoteDir := filepath.Join(dir, "remote")
checkoutDir := filepath.Join(dir, "checkout")
dbRel := filepath.Join("data", "openclaw__openclaw.sync.db")
if err := os.MkdirAll(filepath.Join(remoteDir, "data"), 0o755); err != nil {
t.Fatalf("mkdir remote data: %v", err)
}
if err := runGit(ctx, remoteDir, "init", "-b", "main"); err != nil {
t.Fatalf("git init: %v", err)
}
seedPortableThread(t, filepath.Join(remoteDir, dbRel), 1, "initial issue")
if err := runGit(ctx, remoteDir, "add", dbRel); err != nil {
t.Fatalf("git add seed: %v", err)
}
if err := runGit(ctx, remoteDir, "-c", "user.email=test@example.com", "-c", "user.name=Test", "commit", "-m", "seed store"); err != nil {
t.Fatalf("git commit seed: %v", err)
}
if _, err := syncPortableStore(ctx, remoteDir, checkoutDir); err != nil {
t.Fatalf("clone portable store: %v", err)
}
configPath := filepath.Join(dir, "config.toml")
app := New()
if err := app.Run(ctx, []string{"--config", configPath, "init", "--db", filepath.Join(checkoutDir, dbRel)}); err != nil {
t.Fatalf("init config: %v", err)
}
seedPortableThread(t, filepath.Join(remoteDir, dbRel), 2, "refreshed issue")
if err := runGit(ctx, remoteDir, "add", dbRel); err != nil {
t.Fatalf("git add update: %v", err)
}
if err := runGit(ctx, remoteDir, "-c", "user.email=test@example.com", "-c", "user.name=Test", "commit", "-m", "update store"); err != nil {
t.Fatalf("git commit update: %v", err)
}
run := New()
var stdout bytes.Buffer
run.Stdout = &stdout
if err := run.Run(ctx, []string{"--config", configPath, "threads", "openclaw/openclaw", "--numbers", "2", "--json"}); err != nil {
t.Fatalf("threads: %v", err)
}
if !strings.Contains(stdout.String(), "refreshed issue") {
t.Fatalf("read command did not refresh portable store, got %q", stdout.String())
}
}
func seedPortableThread(t *testing.T, dbPath string, number int, title string) {
t.Helper()
ctx := context.Background()
st, err := store.Open(ctx, dbPath)
if err != nil {
t.Fatalf("open portable db: %v", err)
}
now := time.Now().UTC().Format(time.RFC3339Nano)
repoID, err := st.UpsertRepository(ctx, store.Repository{
Owner: "openclaw",
Name: "openclaw",
FullName: "openclaw/openclaw",
RawJSON: "{}",
UpdatedAt: now,
})
if err != nil {
t.Fatalf("upsert repository: %v", err)
}
if _, err := st.UpsertThread(ctx, store.Thread{
RepoID: repoID,
GitHubID: strconv.Itoa(number),
Number: number,
Kind: "issue",
State: "open",
Title: title,
Body: title,
HTMLURL: fmt.Sprintf("https://github.com/openclaw/openclaw/issues/%d", number),
LabelsJSON: "[]",
AssigneesJSON: "[]",
RawJSON: "{}",
ContentHash: fmt.Sprintf("hash-%d", number),
UpdatedAt: now,
}); err != nil {
t.Fatalf("upsert thread: %v", err)
}
if _, err := st.DB().ExecContext(ctx, `pragma wal_checkpoint(TRUNCATE)`); err != nil {
t.Fatalf("checkpoint portable db: %v", err)
}
if err := st.Close(); err != nil {
t.Fatalf("close portable db: %v", err)
}
}
func TestPortablePruneCommand(t *testing.T) {
dir := t.TempDir()
configPath := filepath.Join(dir, "config.toml")

View File

@ -3,6 +3,8 @@ package cli
import (
"context"
"fmt"
"os"
"path/filepath"
"github.com/openclaw/gitcrawl/internal/config"
"github.com/openclaw/gitcrawl/internal/store"
@ -30,6 +32,7 @@ func (a *App) openLocalRuntimeReadOnly(ctx context.Context) (localRuntime, error
if err != nil {
return localRuntime{}, err
}
_ = refreshPortableStoreForDB(ctx, cfg.DBPath)
st, err := store.OpenReadOnly(ctx, cfg.DBPath)
if err != nil {
return localRuntime{}, err
@ -51,3 +54,38 @@ func (rt localRuntime) defaultRepository(ctx context.Context) (store.Repository,
}
return repos[0], nil
}
func refreshPortableStoreForDB(ctx context.Context, dbPath string) error {
root, ok := portableStoreRoot(dbPath)
if !ok {
return nil
}
if !gitWorktreeClean(ctx, root) {
return nil
}
return runGit(ctx, "", "-C", root, "pull", "--ff-only", "--quiet")
}
func portableStoreRoot(dbPath string) (string, bool) {
dir := filepath.Clean(filepath.Dir(dbPath))
for {
if info, err := os.Stat(filepath.Join(dir, ".git")); err == nil && info.IsDir() {
return dir, true
}
parent := filepath.Dir(dir)
if parent == dir {
return "", false
}
dir = parent
}
}
func gitWorktreeClean(ctx context.Context, dir string) bool {
if err := runGit(ctx, "", "-C", dir, "diff", "--quiet", "--"); err != nil {
return false
}
if err := runGit(ctx, "", "-C", dir, "diff", "--cached", "--quiet", "--"); err != nil {
return false
}
return true
}