openclaw/discrawl
openclaw/discrawl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases 4 Wiki Activity

fix(sync): include progress percentages

Browse Source
This commit is contained in:
Vincent Koc 2026-05-02 19:34:14 -07:00
parent 2e20cb9f11
commit b06fb7f402
No known key found for this signature in database
4 changed files with 21 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -41,7 +41,7 @@ require (
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/vincentkoc/crawlkit v0.3.13
github.com/vincentkoc/crawlkit v0.3.14
golang.org/x/crypto v0.50.0 // indirect
golang.org/x/tools v0.44.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect

2
go.sum
View File

@ -79,6 +79,8 @@ github.com/vincentkoc/crawlkit v0.3.12 h1:2hs4DXk6LkI4sdbgnFU+mUNaC2gmhQfkMx5C+b
github.com/vincentkoc/crawlkit v0.3.12/go.mod h1:tSSR6CmUqKmfoxzxxRJGARm95sH+Acu63nhzrXkpXo0=
github.com/vincentkoc/crawlkit v0.3.13 h1:8QDpI1KXRhvxGlHpomHn641+S3aq7nGLtD8mMLZvCHo=
github.com/vincentkoc/crawlkit v0.3.13/go.mod h1:tSSR6CmUqKmfoxzxxRJGARm95sH+Acu63nhzrXkpXo0=
github.com/vincentkoc/crawlkit v0.3.14 h1:+bE9yPjfE2VRvquJEpHvh+35qcX70RiqisZv/7vChW0=
github.com/vincentkoc/crawlkit v0.3.14/go.mod h1:tSSR6CmUqKmfoxzxxRJGARm95sH+Acu63nhzrXkpXo0=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs=

14
internal/syncer/message_sync.go
View File

@ -7,6 +7,8 @@ import (
"time"
"github.com/bwmarrin/discordgo"
"github.com/vincentkoc/crawlkit/progress"
"github.com/steipete/discrawl/internal/store"
)
@ -663,6 +665,8 @@ func (p *messageSyncProgress) complete(channel *discordgo.Channel, count int, ou
totalChannels := p.totalChannels
messages := p.messages
elapsed := now.Sub(p.startedAt).Round(time.Second).String()
percent := progress.Percent(int64(processed), int64(totalChannels))
completion := progress.Completion(int64(processed), int64(totalChannels))
p.mu.Unlock()
p.syncer.logger.Info(
"message sync progress",
@ -670,6 +674,8 @@ func (p *messageSyncProgress) complete(channel *discordgo.Channel, count int, ou
"processed_channels", processed,
"total_channels", totalChannels,
"remaining_channels", totalChannels-processed,
"percent", percent,
"completion", completion,
"active_channels", activeChannels,
"messages_written", messages,
"deferred_channels", deferred,
@ -698,6 +704,8 @@ func (p *messageSyncProgress) finish(err error) {
totalChannels := p.totalChannels
messages := p.messages
elapsed := now.Sub(p.startedAt).Round(time.Second).String()
percent := progress.Percent(int64(processed), int64(totalChannels))
completion := progress.Completion(int64(processed), int64(totalChannels))
oldestID, oldestName, oldestElapsed, oldestIdle, oldestPages, oldestPageMessages := oldestInflightDetails(p.inflight, now)
p.mu.Unlock()
attrs := []any{
@ -705,6 +713,8 @@ func (p *messageSyncProgress) finish(err error) {
"processed_channels", processed,
"total_channels", totalChannels,
"remaining_channels", totalChannels - processed,
"percent", percent,
"completion", completion,
"active_channels", activeChannels,
"messages_written", messages,
"deferred_channels", deferred,
@ -766,6 +776,8 @@ func (p *messageSyncProgress) logWaitHeartbeat() {
messages := p.messages
idleFor := now.Sub(p.lastProgressAt).Round(time.Second).String()
elapsed := now.Sub(p.startedAt).Round(time.Second).String()
percent := progress.Percent(int64(processed), int64(totalChannels))
completion := progress.Completion(int64(processed), int64(totalChannels))
oldestID, oldestName, oldestElapsed, oldestIdle, oldestPages, oldestPageMessages := oldestInflightDetails(p.inflight, now)
p.mu.Unlock()
p.syncer.logger.Info(
@ -774,6 +786,8 @@ func (p *messageSyncProgress) logWaitHeartbeat() {
"processed_channels", processed,
"total_channels", totalChannels,
"remaining_channels", totalChannels-processed,
"percent", percent,
"completion", completion,
"active_channels", activeChannels,
"messages_written", messages,
"deferred_channels", deferred,

4
internal/syncer/message_sync_progress_test.go
View File

@ -73,6 +73,8 @@ func TestMessageSyncProgressFinishReportsSummaryCounts(t *testing.T) {
logs := out.String()
require.Contains(t, logs, `msg="message sync finished"`)
require.Contains(t, logs, `processed_channels=3`)
require.Contains(t, logs, `percent=100.0`)
require.Contains(t, logs, `completion=100.0%`)
require.Contains(t, logs, `messages_written=42`)
require.Contains(t, logs, `skipped_missing_access_channels=1`)
require.Contains(t, logs, `skipped_unknown_channel_channels=1`)
@ -105,4 +107,6 @@ func TestMessageSyncProgressReportsWaitingHeartbeat(t *testing.T) {
require.Contains(t, logs, `oldest_active_channel_id=c1`)
require.Contains(t, logs, `oldest_active_channel_name=slowpoke`)
require.Contains(t, logs, `active_channels=1`)
require.Contains(t, logs, `percent=0.0`)
require.Contains(t, logs, `completion=0.0%`)
}