diff --git a/go.sum b/go.sum index 2402a81..a2b2cf6 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,8 @@ github.com/vincentkoc/crawlkit v0.3.12 h1:2hs4DXk6LkI4sdbgnFU+mUNaC2gmhQfkMx5C+b github.com/vincentkoc/crawlkit v0.3.12/go.mod h1:tSSR6CmUqKmfoxzxxRJGARm95sH+Acu63nhzrXkpXo0= github.com/vincentkoc/crawlkit v0.3.13 h1:8QDpI1KXRhvxGlHpomHn641+S3aq7nGLtD8mMLZvCHo= github.com/vincentkoc/crawlkit v0.3.13/go.mod h1:tSSR6CmUqKmfoxzxxRJGARm95sH+Acu63nhzrXkpXo0= +github.com/vincentkoc/crawlkit v0.3.14 h1:+bE9yPjfE2VRvquJEpHvh+35qcX70RiqisZv/7vChW0= +github.com/vincentkoc/crawlkit v0.3.14/go.mod h1:tSSR6CmUqKmfoxzxxRJGARm95sH+Acu63nhzrXkpXo0= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= diff --git a/internal/cli/app.go b/internal/cli/app.go index ca9bcf6..3551577 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -7,6 +7,7 @@ import ( "flag" "fmt" "io" + "log/slog" "os" "os/exec" "path/filepath" @@ -1821,6 +1822,7 @@ func (a *App) syncRepository(ctx context.Context, owner, repo string, options sy Reporter: func(message string) { fmt.Fprintln(a.Stderr, message) }, + Logger: progressLogger(a.Stderr), }) if err != nil { return syncer.Stats{}, err @@ -1828,6 +1830,17 @@ func (a *App) syncRepository(ctx context.Context, owner, repo string, options sy return stats, nil } +func progressLogger(w io.Writer) *slog.Logger { + return slog.New(slog.NewTextHandler(w, &slog.HandlerOptions{ + ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr { + if attr.Key == slog.TimeKey { + return slog.Attr{} + } + return attr + }, + })) +} + func (a *App) runInit(ctx context.Context, args []string) error { fs := flag.NewFlagSet("init", flag.ContinueOnError) fs.SetOutput(io.Discard) diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index bb47944..d41c8a3 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "log/slog" "strconv" "strings" "time" @@ -13,6 +14,7 @@ import ( "github.com/openclaw/gitcrawl/internal/documents" gh "github.com/openclaw/gitcrawl/internal/github" "github.com/openclaw/gitcrawl/internal/store" + "github.com/vincentkoc/crawlkit/progress" ) type GitHubClient interface { @@ -45,6 +47,7 @@ type Options struct { IncludeComments bool IncludePRDetails bool Reporter gh.Reporter + Logger *slog.Logger } type Stats struct { @@ -132,6 +135,15 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { MetadataOnly: !options.IncludeComments, StartedAt: started, } + tracker := progress.New(options.Logger, progress.Options{ + Name: "sync", + Unit: "threads", + Total: int64(len(rows)), + Attrs: []any{ + "repository", stats.Repository, + "state", state, + }, + }) persist := func(st *store.Store) error { for _, row := range rows { thread := mapIssueToThread(repoID, row, s.now().Format(time.RFC3339Nano)) @@ -169,6 +181,11 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { } else { stats.IssuesSynced++ } + tracker.Add(1, + "number", thread.Number, + "kind", thread.Kind, + "thread_state", thread.State, + ) } if len(numbers) == 0 && state == "open" && since != "" && options.Limit <= 0 { closed, err := s.applyClosedOverlapSweep(ctx, st, repoID, options, since) @@ -193,13 +210,17 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { } if !options.IncludeComments { if err := s.store.WithTx(ctx, persist); err != nil { + tracker.Finish(err) return Stats{}, err } + tracker.Finish(nil) return stats, nil } if err := persist(s.store); err != nil { + tracker.Finish(err) return Stats{}, err } + tracker.Finish(nil) return stats, nil } diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index 7a2a606..8ac8c85 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -1,9 +1,12 @@ package syncer import ( + "bytes" "context" "encoding/json" + "log/slog" "path/filepath" + "strings" "testing" "time" @@ -286,7 +289,13 @@ func TestSyncPersistsIssuesAndPullRequests(t *testing.T) { s := New(fakeGitHub{}, st) s.now = func() time.Time { return time.Date(2026, 4, 26, 0, 0, 0, 0, time.UTC) } - stats, err := s.Sync(ctx, Options{Owner: "openclaw", Repo: "gitcrawl", IncludeComments: true}) + var progressLogs bytes.Buffer + stats, err := s.Sync(ctx, Options{ + Owner: "openclaw", + Repo: "gitcrawl", + IncludeComments: true, + Logger: testProgressLogger(&progressLogs), + }) if err != nil { t.Fatalf("sync: %v", err) } @@ -321,6 +330,18 @@ func TestSyncPersistsIssuesAndPullRequests(t *testing.T) { if documentCount != 1 { t.Fatalf("document count: got %d want 1", documentCount) } + for _, want := range []string{ + `msg="sync progress"`, + `state=finished`, + `unit=threads`, + `percent=100.0`, + `completion=100.0%`, + `repository=openclaw/gitcrawl`, + } { + if !strings.Contains(progressLogs.String(), want) { + t.Fatalf("missing %q in progress logs:\n%s", want, progressLogs.String()) + } + } } func TestSyncHydratesPullReviewComments(t *testing.T) { @@ -681,3 +702,14 @@ func TestMappingFallbackBranches(t *testing.T) { t.Fatalf("thread = %+v", thread) } } + +func testProgressLogger(out *bytes.Buffer) *slog.Logger { + return slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{ + ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr { + if attr.Key == slog.TimeKey { + return slog.Attr{} + } + return attr + }, + })) +}