openclaw/gitcrawl
openclaw/gitcrawl
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 7 Wiki Activity

fix(tui): refresh remote stores atomically

Browse Source
This commit is contained in:
Vincent Koc 2026-04-28 21:14:26 -07:00
parent 6678a14bb0
commit 40ac3358e4
No known key found for this signature in database
5 changed files with 274 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
internal/cli/app.go
View File

@ -977,8 +977,10 @@ func (a *App) runTUI(ctx context.Context, args []string) error {
Repository: repo.FullName,
InferredRepository: inferred,
Mode: "cluster-browser",
DBSource: databaseSourceKind(rt.Config.DBPath),
DBLocation: databaseSourceLocation(ctx, rt.Config.DBPath),
DBSource: databaseSourceKind(rt.SourceDBPath),
DBLocation: databaseSourceLocation(ctx, rt.SourceDBPath),
DBRefreshSource: remoteRefreshSource(rt),
DBRuntimePath: remoteRuntimePath(rt),
Sort: sort,
MinSize: minSize,
Limit: limit,
@ -1003,6 +1005,20 @@ func databaseSourceKind(dbPath string) string {
return "local"
}
func remoteRefreshSource(rt localRuntime) string {
if rt.RemoteSource {
return rt.SourceDBPath
}
return ""
}
func remoteRuntimePath(rt localRuntime) string {
if rt.RemoteSource {
return rt.Config.DBPath
}
return ""
}
func databaseSourceLocation(ctx context.Context, dbPath string) string {
filename := filepath.Base(dbPath)
root, ok := portableStoreRoot(dbPath)

10
internal/cli/app_test.go
View File

@ -218,6 +218,16 @@ func TestReadCommandRefreshesPortableStore(t *testing.T) {
if !strings.Contains(stdout.String(), "refreshed issue") {
t.Fatalf("read command did not refresh portable store, got %q", stdout.String())
}
if !gitWorktreeClean(ctx, checkoutDir) {
t.Fatal("portable checkout should stay clean after read-only command")
}
mirrorPath, err := run.portableRuntimeDBPath(filepath.Join(checkoutDir, dbRel))
if err != nil {
t.Fatalf("runtime db path: %v", err)
}
if _, err := os.Stat(mirrorPath); err != nil {
t.Fatalf("runtime mirror db was not created: %v", err)
}
}
func seedPortableThread(t *testing.T, dbPath string, number int, title string) {

146
internal/cli/runtime.go
View File

@ -3,16 +3,21 @@ package cli
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"github.com/openclaw/gitcrawl/internal/config"
"github.com/openclaw/gitcrawl/internal/store"
)
type localRuntime struct {
Config config.Config
Store *store.Store
Config config.Config
Store *store.Store
SourceDBPath string
RemoteSource bool
}
func (a *App) openLocalRuntime(ctx context.Context) (localRuntime, error) {
@ -20,11 +25,28 @@ func (a *App) openLocalRuntime(ctx context.Context) (localRuntime, error) {
if err != nil {
return localRuntime{}, err
}
st, err := store.Open(ctx, cfg.DBPath)
sourceDBPath := cfg.DBPath
remoteSource := false
openReadOnly := false
if _, ok := portableStoreRoot(cfg.DBPath); ok {
mirrorPath, _, err := a.ensurePortableRuntimeDB(ctx, cfg.DBPath, false)
if err != nil {
return localRuntime{}, err
}
cfg.DBPath = mirrorPath
remoteSource = true
openReadOnly = true
}
var st *store.Store
if openReadOnly {
st, err = store.OpenReadOnly(ctx, cfg.DBPath)
} else {
st, err = store.Open(ctx, cfg.DBPath)
}
if err != nil {
return localRuntime{}, err
}
return localRuntime{Config: cfg, Store: st}, nil
return localRuntime{Config: cfg, Store: st, SourceDBPath: sourceDBPath, RemoteSource: remoteSource}, nil
}
func (a *App) openLocalRuntimeReadOnly(ctx context.Context) (localRuntime, error) {
@ -32,12 +54,21 @@ func (a *App) openLocalRuntimeReadOnly(ctx context.Context) (localRuntime, error
if err != nil {
return localRuntime{}, err
}
_ = refreshPortableStoreForDB(ctx, cfg.DBPath)
sourceDBPath := cfg.DBPath
remoteSource := false
if _, ok := portableStoreRoot(cfg.DBPath); ok {
mirrorPath, _, err := a.ensurePortableRuntimeDB(ctx, cfg.DBPath, true)
if err != nil {
return localRuntime{}, err
}
cfg.DBPath = mirrorPath
remoteSource = true
}
st, err := store.OpenReadOnly(ctx, cfg.DBPath)
if err != nil {
return localRuntime{}, err
}
return localRuntime{Config: cfg, Store: st}, nil
return localRuntime{Config: cfg, Store: st, SourceDBPath: sourceDBPath, RemoteSource: remoteSource}, nil
}
func (rt localRuntime) repository(ctx context.Context, owner, repo string) (store.Repository, error) {
@ -66,6 +97,109 @@ func refreshPortableStoreForDB(ctx context.Context, dbPath string) error {
return runGit(ctx, "", "-C", root, "pull", "--ff-only", "--quiet")
}
var portableRuntimeMu sync.Mutex
func (a *App) ensurePortableRuntimeDB(ctx context.Context, sourceDBPath string, refresh bool) (string, bool, error) {
mirrorPath, err := a.portableRuntimeDBPath(sourceDBPath)
if err != nil {
return "", false, err
}
changed, err := refreshPortableRuntimeDB(ctx, sourceDBPath, mirrorPath, refresh)
return mirrorPath, changed, err
}
func (a *App) portableRuntimeDBPath(sourceDBPath string) (string, error) {
root, ok := portableStoreRoot(sourceDBPath)
if !ok {
return "", fmt.Errorf("portable store root not found for %s", sourceDBPath)
}
rel, err := filepath.Rel(root, sourceDBPath)
if err != nil || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) || rel == ".." || filepath.IsAbs(rel) {
return "", fmt.Errorf("portable database %s is outside store root %s", sourceDBPath, root)
}
name := safePathName(filepath.Base(root))
if name == "" {
name = "portable-store"
}
return filepath.Join(filepath.Dir(config.ResolvePath(a.configPath)), "runtime", name, rel), nil
}
func refreshPortableRuntimeDB(ctx context.Context, sourceDBPath, mirrorPath string, refresh bool) (bool, error) {
portableRuntimeMu.Lock()
defer portableRuntimeMu.Unlock()
if refresh {
if err := refreshPortableStoreForDB(ctx, sourceDBPath); err != nil {
return false, err
}
}
needsCopy, err := portableRuntimeNeedsCopy(sourceDBPath, mirrorPath)
if err != nil {
return false, err
}
if !needsCopy {
return false, nil
}
if err := copyFileAtomic(sourceDBPath, mirrorPath); err != nil {
return false, err
}
return true, nil
}
func portableRuntimeNeedsCopy(sourceDBPath, mirrorPath string) (bool, error) {
sourceInfo, err := os.Stat(sourceDBPath)
if err != nil {
return false, fmt.Errorf("stat portable source db: %w", err)
}
mirrorInfo, err := os.Stat(mirrorPath)
if err != nil {
if os.IsNotExist(err) {
return true, nil
}
return false, fmt.Errorf("stat portable runtime db: %w", err)
}
return sourceInfo.Size() != mirrorInfo.Size() || sourceInfo.ModTime().After(mirrorInfo.ModTime()), nil
}
func copyFileAtomic(sourcePath, targetPath string) error {
if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil {
return fmt.Errorf("create portable runtime dir: %w", err)
}
source, err := os.Open(sourcePath)
if err != nil {
return fmt.Errorf("open portable source db: %w", err)
}
defer source.Close()
temp, err := os.CreateTemp(filepath.Dir(targetPath), "."+filepath.Base(targetPath)+".tmp-*")
if err != nil {
return fmt.Errorf("create portable runtime temp db: %w", err)
}
tempPath := temp.Name()
cleanup := true
defer func() {
if cleanup {
_ = os.Remove(tempPath)
}
}()
if _, err := io.Copy(temp, source); err != nil {
_ = temp.Close()
return fmt.Errorf("copy portable runtime db: %w", err)
}
if err := temp.Chmod(0o600); err != nil {
_ = temp.Close()
return fmt.Errorf("chmod portable runtime db: %w", err)
}
if err := temp.Close(); err != nil {
return fmt.Errorf("close portable runtime db: %w", err)
}
if err := os.Rename(tempPath, targetPath); err != nil {
return fmt.Errorf("replace portable runtime db: %w", err)
}
cleanup = false
_ = os.Remove(targetPath + "-wal")
_ = os.Remove(targetPath + "-shm")
return nil
}
func portableStoreRoot(dbPath string) (string, bool) {
dir := filepath.Clean(filepath.Dir(dbPath))
for {

88
internal/cli/tui.go
View File

@ -35,6 +35,12 @@ var (
const tuiAutoRefreshInterval = 15 * time.Second
type tuiAutoRefreshMsg struct{}
type tuiRemoteRefreshTickMsg struct{}
type tuiRemoteRefreshMsg struct {
changed bool
err error
}
type clusterBrowserPayload struct {
Repository string `json:"repository"`
@ -42,6 +48,8 @@ type clusterBrowserPayload struct {
Mode string `json:"mode"`
DBSource string `json:"db_source,omitempty"`
DBLocation string `json:"db_location,omitempty"`
DBRefreshSource string `json:"-"`
DBRuntimePath string `json:"-"`
Sort string `json:"sort"`
MinSize int `json:"min_size"`
Limit int `json:"limit,omitempty"`
@ -120,6 +128,8 @@ type clusterBrowserModel struct {
neighborCache map[int64][]tuiNeighbor
detail store.ClusterDetail
hasDetail bool
remoteRefreshing bool
remoteFrame int
}
type memberRow struct {
@ -173,7 +183,10 @@ func (a *App) runInteractiveTUI(ctx context.Context, st *store.Store, repoID int
}
model := newClusterBrowserModel(ctx, st, repoID, payload)
program := tea.NewProgram(model, tea.WithInput(os.Stdin), tea.WithOutput(out), tea.WithAltScreen(), tea.WithMouseCellMotion())
_, err := program.Run()
finalModel, err := program.Run()
if final, ok := finalModel.(clusterBrowserModel); ok && final.store != nil && final.store != st {
_ = final.store.Close()
}
return err
}
@ -203,13 +216,21 @@ func newClusterBrowserModel(ctx context.Context, st *store.Store, repoID int64,
detailCache: map[int64]store.ClusterDetail{},
neighborCache: map[int64][]tuiNeighbor{},
}
if payload.DBSource == "remote" && payload.DBRefreshSource != "" && payload.DBRuntimePath != "" {
model.remoteRefreshing = true
model.status = "Refreshing remote data"
}
model.applyClusterFilters()
model.loadSelectedCluster()
return model
}
func (m clusterBrowserModel) Init() tea.Cmd {
return m.autoRefreshCmd()
cmds := []tea.Cmd{m.autoRefreshCmd()}
if m.remoteRefreshing {
cmds = append(cmds, m.remoteRefreshCmd(), m.remoteRefreshTickCmd())
}
return tea.Batch(cmds...)
}
func (m clusterBrowserModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
@ -220,6 +241,29 @@ func (m clusterBrowserModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
}
m.autoRefreshFromStore()
return m, m.autoRefreshCmd()
case tuiRemoteRefreshTickMsg:
if !m.remoteRefreshing {
return m, nil
}
m.remoteFrame++
return m, m.remoteRefreshTickCmd()
case tuiRemoteRefreshMsg:
m.remoteRefreshing = false
if msg.err != nil {
m.status = "Remote refresh failed: " + msg.err.Error()
return m, nil
}
if msg.changed {
if err := m.reopenRuntimeStore(); err != nil {
m.status = "Remote refresh loaded but reopen failed: " + err.Error()
return m, nil
}
m.refreshFromStore()
m.status = "Remote data refreshed"
return m, nil
}
m.status = "Remote data already current"
return m, nil
case tea.WindowSizeMsg:
m.width = msg.Width
m.height = msg.Height
@ -335,6 +379,38 @@ func (m clusterBrowserModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
return m, nil
}
func (m clusterBrowserModel) remoteRefreshCmd() tea.Cmd {
sourceDBPath := m.payload.DBRefreshSource
runtimeDBPath := m.payload.DBRuntimePath
return func() tea.Msg {
changed, err := refreshPortableRuntimeDB(m.ctx, sourceDBPath, runtimeDBPath, true)
return tuiRemoteRefreshMsg{changed: changed, err: err}
}
}
func (m clusterBrowserModel) remoteRefreshTickCmd() tea.Cmd {
return tea.Tick(120*time.Millisecond, func(time.Time) tea.Msg {
return tuiRemoteRefreshTickMsg{}
})
}
func (m *clusterBrowserModel) reopenRuntimeStore() error {
if strings.TrimSpace(m.payload.DBRuntimePath) == "" {
return nil
}
next, err := store.OpenReadOnly(m.ctx, m.payload.DBRuntimePath)
if err != nil {
return err
}
if m.store != nil {
_ = m.store.Close()
}
m.store = next
m.detailCache = map[int64]store.ClusterDetail{}
m.neighborCache = map[int64][]tuiNeighbor{}
return nil
}
func (m clusterBrowserModel) View() string {
if m.width <= 0 || m.height <= 0 {
return "loading gitcrawl tui..."
@ -457,6 +533,9 @@ func (m clusterBrowserModel) renderFooter(width int) string {
if m.jumping {
line = "Jump: " + m.searchInput.View()
}
if m.remoteRefreshing {
line = fmt.Sprintf("Refreshing remote data %s %s", loadingFrame(m.remoteFrame), line)
}
if location := m.footerLocation(); location != "" {
line = strings.TrimSpace(line + " " + location)
}
@ -464,6 +543,11 @@ func (m clusterBrowserModel) renderFooter(width int) string {
return lipgloss.NewStyle().Width(width).Height(2).Background(bg).Foreground(fg).Padding(0, 1).Render(truncateCells(line, width-2) + "\n" + truncateCells(controls, maxInt(1, width-2)))
}
func loadingFrame(index int) string {
frames := []string{"-", "\\", "|", "/"}
return frames[index%len(frames)]
}
func (m clusterBrowserModel) footerLocation() string {
location := strings.TrimSpace(m.payload.DBLocation)
if location == "" {

20
internal/cli/tui_test.go
View File

@ -131,6 +131,26 @@ func TestTUIFooterShowsRemoteDatabaseLocation(t *testing.T) {
}
}
func TestTUIFooterShowsRemoteRefreshLoadingState(t *testing.T) {
model := newClusterBrowserModel(context.Background(), nil, 0, clusterBrowserPayload{
Repository: "openclaw/openclaw",
DBSource: "remote",
DBLocation: "openclaw/gitcrawl-store:openclaw__openclaw.sync.db",
DBRefreshSource: "/tmp/source.db",
DBRuntimePath: "/tmp/runtime.db",
Sort: "recent",
Clusters: sampleTUIClusters(),
})
if !model.remoteRefreshing {
t.Fatal("remote model should start in refresh loading state")
}
footer := model.renderFooter(140)
if !strings.Contains(footer, "Refreshing remote data") {
t.Fatalf("footer missing remote refresh loading state:\n%s", footer)
}
}
func TestTUIViewFitsTerminalFrame(t *testing.T) {
model := newClusterBrowserModel(context.Background(), nil, 0, clusterBrowserPayload{
Repository: "openclaw/openclaw",