From b06fb7f402cf2b769732b6dcac8136ef37c332d1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 2 May 2026 19:34:14 -0700 Subject: [PATCH] fix(sync): include progress percentages --- go.mod | 2 +- go.sum | 2 ++ internal/syncer/message_sync.go | 14 ++++++++++++++ internal/syncer/message_sync_progress_test.go | 4 ++++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5ac4bca..fa9652c 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/ncruces/go-strftime v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/vincentkoc/crawlkit v0.3.13 + github.com/vincentkoc/crawlkit v0.3.14 golang.org/x/crypto v0.50.0 // indirect golang.org/x/tools v0.44.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index 51a7a3e..5b86600 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,8 @@ github.com/vincentkoc/crawlkit v0.3.12 h1:2hs4DXk6LkI4sdbgnFU+mUNaC2gmhQfkMx5C+b github.com/vincentkoc/crawlkit v0.3.12/go.mod h1:tSSR6CmUqKmfoxzxxRJGARm95sH+Acu63nhzrXkpXo0= github.com/vincentkoc/crawlkit v0.3.13 h1:8QDpI1KXRhvxGlHpomHn641+S3aq7nGLtD8mMLZvCHo= github.com/vincentkoc/crawlkit v0.3.13/go.mod h1:tSSR6CmUqKmfoxzxxRJGARm95sH+Acu63nhzrXkpXo0= +github.com/vincentkoc/crawlkit v0.3.14 h1:+bE9yPjfE2VRvquJEpHvh+35qcX70RiqisZv/7vChW0= +github.com/vincentkoc/crawlkit v0.3.14/go.mod h1:tSSR6CmUqKmfoxzxxRJGARm95sH+Acu63nhzrXkpXo0= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs= diff --git a/internal/syncer/message_sync.go b/internal/syncer/message_sync.go index 4f61a38..9ded573 100644 --- a/internal/syncer/message_sync.go +++ b/internal/syncer/message_sync.go @@ -7,6 +7,8 @@ import ( "time" "github.com/bwmarrin/discordgo" + "github.com/vincentkoc/crawlkit/progress" + "github.com/steipete/discrawl/internal/store" ) @@ -663,6 +665,8 @@ func (p *messageSyncProgress) complete(channel *discordgo.Channel, count int, ou totalChannels := p.totalChannels messages := p.messages elapsed := now.Sub(p.startedAt).Round(time.Second).String() + percent := progress.Percent(int64(processed), int64(totalChannels)) + completion := progress.Completion(int64(processed), int64(totalChannels)) p.mu.Unlock() p.syncer.logger.Info( "message sync progress", @@ -670,6 +674,8 @@ func (p *messageSyncProgress) complete(channel *discordgo.Channel, count int, ou "processed_channels", processed, "total_channels", totalChannels, "remaining_channels", totalChannels-processed, + "percent", percent, + "completion", completion, "active_channels", activeChannels, "messages_written", messages, "deferred_channels", deferred, @@ -698,6 +704,8 @@ func (p *messageSyncProgress) finish(err error) { totalChannels := p.totalChannels messages := p.messages elapsed := now.Sub(p.startedAt).Round(time.Second).String() + percent := progress.Percent(int64(processed), int64(totalChannels)) + completion := progress.Completion(int64(processed), int64(totalChannels)) oldestID, oldestName, oldestElapsed, oldestIdle, oldestPages, oldestPageMessages := oldestInflightDetails(p.inflight, now) p.mu.Unlock() attrs := []any{ @@ -705,6 +713,8 @@ func (p *messageSyncProgress) finish(err error) { "processed_channels", processed, "total_channels", totalChannels, "remaining_channels", totalChannels - processed, + "percent", percent, + "completion", completion, "active_channels", activeChannels, "messages_written", messages, "deferred_channels", deferred, @@ -766,6 +776,8 @@ func (p *messageSyncProgress) logWaitHeartbeat() { messages := p.messages idleFor := now.Sub(p.lastProgressAt).Round(time.Second).String() elapsed := now.Sub(p.startedAt).Round(time.Second).String() + percent := progress.Percent(int64(processed), int64(totalChannels)) + completion := progress.Completion(int64(processed), int64(totalChannels)) oldestID, oldestName, oldestElapsed, oldestIdle, oldestPages, oldestPageMessages := oldestInflightDetails(p.inflight, now) p.mu.Unlock() p.syncer.logger.Info( @@ -774,6 +786,8 @@ func (p *messageSyncProgress) logWaitHeartbeat() { "processed_channels", processed, "total_channels", totalChannels, "remaining_channels", totalChannels-processed, + "percent", percent, + "completion", completion, "active_channels", activeChannels, "messages_written", messages, "deferred_channels", deferred, diff --git a/internal/syncer/message_sync_progress_test.go b/internal/syncer/message_sync_progress_test.go index e266a9b..436a572 100644 --- a/internal/syncer/message_sync_progress_test.go +++ b/internal/syncer/message_sync_progress_test.go @@ -73,6 +73,8 @@ func TestMessageSyncProgressFinishReportsSummaryCounts(t *testing.T) { logs := out.String() require.Contains(t, logs, `msg="message sync finished"`) require.Contains(t, logs, `processed_channels=3`) + require.Contains(t, logs, `percent=100.0`) + require.Contains(t, logs, `completion=100.0%`) require.Contains(t, logs, `messages_written=42`) require.Contains(t, logs, `skipped_missing_access_channels=1`) require.Contains(t, logs, `skipped_unknown_channel_channels=1`) @@ -105,4 +107,6 @@ func TestMessageSyncProgressReportsWaitingHeartbeat(t *testing.T) { require.Contains(t, logs, `oldest_active_channel_id=c1`) require.Contains(t, logs, `oldest_active_channel_name=slowpoke`) require.Contains(t, logs, `active_channels=1`) + require.Contains(t, logs, `percent=0.0`) + require.Contains(t, logs, `completion=0.0%`) }