openclaw/discrawl
openclaw/discrawl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases 4 Wiki Activity

refactor: use crawlkit package nouns

Browse Source
This commit is contained in:
Vincent Koc 2026-05-01 12:40:28 -07:00
parent a13447fd47
commit 7fcb7bb599
No known key found for this signature in database
7 changed files with 31 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -30,7 +30,7 @@
### Changes
- Add `discrawl tui`, a terminal archive browser for stored guild messages and local `@me` wiretap DMs using the shared `crawlkit/termkit` package.
- Add `discrawl tui`, a terminal archive browser for stored guild messages and local `@me` wiretap DMs using the shared `crawlkit/tui` package.
### Fixes

2
go.mod
View File

@ -41,7 +41,7 @@ require (
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/vincentkoc/crawlkit v0.2.0
github.com/vincentkoc/crawlkit v0.3.0
golang.org/x/crypto v0.50.0 // indirect
golang.org/x/tools v0.44.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect

2
go.sum
View File

@ -75,6 +75,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/vincentkoc/crawlkit v0.2.0 h1:w6jfcckVsKOcx6fFwKj9mvyTJHmSDg3ro+j2/TgNkuE=
github.com/vincentkoc/crawlkit v0.2.0/go.mod h1:Zp6k0f6owZ81wccG26jPbLSDGmfjoxPdzgPXZcUpmW4=
github.com/vincentkoc/crawlkit v0.3.0 h1:37QsucIaUmaRygk2fn7NQniD6YSAAi0DR1vzL8UimoI=
github.com/vincentkoc/crawlkit v0.3.0/go.mod h1:Zp6k0f6owZ81wccG26jPbLSDGmfjoxPdzgPXZcUpmW4=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs=

12
internal/cli/tui_commands.go
View File

@ -7,7 +7,7 @@ import (
"io"
"strings"
"github.com/vincentkoc/crawlkit/termkit"
"github.com/vincentkoc/crawlkit/tui"
"github.com/steipete/discrawl/internal/store"
)
@ -49,12 +49,12 @@ func (r *runtime) runTUI(args []string) error {
if r.json {
return r.print(items)
}
if err := termkit.Run(r.ctx, termkit.Options{
if err := tui.Run(r.ctx, tui.Options{
Title: "discrawl archive",
EmptyMessage: "discrawl has no local messages yet",
Items: items,
}); err != nil {
if errors.Is(err, termkit.ErrNotTerminal) {
if errors.Is(err, tui.ErrNotTerminal) {
return fmt.Errorf("%w; run discrawl tui from a TTY or pass --json", err)
}
return err
@ -62,8 +62,8 @@ func (r *runtime) runTUI(args []string) error {
return nil
}
func discordTUIItems(rows []store.MessageRow) []termkit.Item {
items := make([]termkit.Item, 0, len(rows))
func discordTUIItems(rows []store.MessageRow) []tui.Item {
items := make([]tui.Item, 0, len(rows))
for _, row := range rows {
title := strings.TrimSpace(row.Content)
if title == "" {
@ -73,7 +73,7 @@ func discordTUIItems(rows []store.MessageRow) []termkit.Item {
if row.GuildID == "@me" {
tags = append(tags, "dm")
}
items = append(items, termkit.Item{
items = append(items, tui.Item{
Title: title,
Subtitle: strings.TrimSpace(strings.Join([]string{row.GuildID, row.ChannelName, row.AuthorName, formatTime(row.CreatedAt)}, " ")),
Detail: strings.TrimSpace(strings.Join([]string{

14
internal/config/config.go
View File

@ -9,7 +9,7 @@ import (
"strings"
"time"
"github.com/vincentkoc/crawlkit/configkit"
crawlconfig "github.com/vincentkoc/crawlkit/config"
)
const (
@ -85,14 +85,14 @@ type TokenResolution struct {
Path string
}
var appConfig = configkit.App{Name: "discrawl", ConfigEnv: DefaultConfigEnv, BaseDir: "~/.discrawl", LegacyBaseDir: "~/.discrawl"}
var appConfig = crawlconfig.App{Name: "discrawl", ConfigEnv: DefaultConfigEnv, BaseDir: "~/.discrawl", LegacyBaseDir: "~/.discrawl"}
func Default() Config {
home, _ := os.UserHomeDir()
paths, err := appConfig.DefaultPaths()
if err != nil {
base := filepath.Join(home, ".discrawl")
paths = configkit.Paths{
paths = crawlconfig.Paths{
DBPath: filepath.Join(base, "discrawl.db"),
CacheDir: filepath.Join(base, "cache"),
LogDir: filepath.Join(base, "logs"),
@ -170,7 +170,7 @@ func Load(path string) (Config, error) {
if err != nil {
return Config{}, err
}
if err := configkit.LoadTOML(expanded, &cfg); err != nil {
if err := crawlconfig.LoadTOML(expanded, &cfg); err != nil {
return Config{}, err
}
if err := cfg.Normalize(); err != nil {
@ -187,7 +187,7 @@ func Write(path string, cfg Config) error {
if err != nil {
return err
}
return configkit.WriteTOML(expanded, cfg, 0o600)
return crawlconfig.WriteTOML(expanded, cfg, 0o600)
}
func (c *Config) Normalize() error {
@ -341,7 +341,7 @@ func (c Config) ShareEnabled() bool {
}
func EnsureRuntimeDirs(cfg Config) error {
return configkit.EnsureRuntimeDirs(configkit.RuntimeConfig{
return crawlconfig.EnsureRuntimeDirs(crawlconfig.RuntimeConfig{
DBPath: cfg.DBPath,
CacheDir: cfg.CacheDir,
LogDir: cfg.LogDir,
@ -352,7 +352,7 @@ func ExpandPath(path string) (string, error) {
if strings.TrimSpace(path) == "" {
return "", errors.New("empty path")
}
return filepath.Clean(os.ExpandEnv(configkit.ExpandHome(path))), nil
return filepath.Clean(os.ExpandEnv(crawlconfig.ExpandHome(path))), nil
}
func uniqueStrings(in []string) []string {

24
internal/share/share.go
View File

@ -17,8 +17,8 @@ import (
"time"
"github.com/openclaw/discrawl/internal/store"
"github.com/vincentkoc/crawlkit/gitshare"
"github.com/vincentkoc/crawlkit/pack"
"github.com/vincentkoc/crawlkit/mirror"
"github.com/vincentkoc/crawlkit/snapshot"
)
const (
@ -28,7 +28,7 @@ const (
directMessageGuildID = "@me"
)
var ErrNoManifest = pack.ErrNoManifest
var ErrNoManifest = snapshot.ErrNoManifest
const shardFlushRows = 1024
@ -74,7 +74,7 @@ type Manifest struct {
Files map[string]string `json:"files,omitempty"`
}
type TableManifest = pack.TableManifest
type TableManifest = snapshot.TableManifest
type EmbeddingManifest struct {
Provider string `json:"provider"`
@ -89,22 +89,22 @@ func EnsureRepo(ctx context.Context, opts Options) error {
if strings.TrimSpace(opts.RepoPath) == "" {
return errors.New("share repo path is empty")
}
return gitshare.EnsureRepo(ctx, gitshareOptions(opts))
return mirror.EnsureRepo(ctx, mirrorOptions(opts))
}
func Pull(ctx context.Context, opts Options) error {
if strings.TrimSpace(opts.Remote) == "" && strings.TrimSpace(opts.RepoPath) == "" {
return nil
}
return gitshare.Pull(ctx, gitshareOptions(opts))
return mirror.Pull(ctx, mirrorOptions(opts))
}
func Commit(ctx context.Context, opts Options, message string) (bool, error) {
return gitshare.Commit(ctx, gitshareOptions(opts), message)
return mirror.Commit(ctx, mirrorOptions(opts), message)
}
func Push(ctx context.Context, opts Options) error {
if err := gitshare.Push(ctx, gitshareOptions(opts)); err != nil {
if err := mirror.Push(ctx, mirrorOptions(opts)); err != nil {
branch := opts.Branch
if strings.TrimSpace(branch) == "" {
branch = "main"
@ -118,7 +118,7 @@ func Export(ctx context.Context, s *store.Store, opts Options) (Manifest, error)
if err := EnsureRepo(ctx, opts); err != nil {
return Manifest{}, err
}
base, err := pack.Export(ctx, pack.ExportOptions{
base, err := snapshot.Export(ctx, snapshot.ExportOptions{
DB: s.DB(),
RootDir: opts.RepoPath,
Tables: SnapshotTables,
@ -170,7 +170,7 @@ func Import(ctx context.Context, s *store.Store, opts Options) (Manifest, error)
_ = restorePragmas(ctx)
}
}()
if _, err := pack.Import(ctx, pack.ImportOptions{
if _, err := snapshot.Import(ctx, snapshot.ImportOptions{
DB: s.DB(),
RootDir: opts.RepoPath,
DeleteTables: SnapshotTables,
@ -347,8 +347,8 @@ func ReadManifest(repoPath string) (Manifest, error) {
return manifest, nil
}
func gitshareOptions(opts Options) gitshare.Options {
return gitshare.Options{RepoPath: opts.RepoPath, Remote: opts.Remote, Branch: opts.Branch}
func mirrorOptions(opts Options) mirror.Options {
return mirror.Options{RepoPath: opts.RepoPath, Remote: opts.Remote, Branch: opts.Branch}
}
func NeedsImport(ctx context.Context, s *store.Store, staleAfter time.Duration) bool {

4
internal/store/store.go
View File

@ -8,7 +8,7 @@ import (
"strconv"
"time"
"github.com/vincentkoc/crawlkit/sqlitekit"
crawlstore "github.com/vincentkoc/crawlkit/store"
)
const (
@ -111,7 +111,7 @@ type ChannelRow struct {
}
func Open(ctx context.Context, path string) (*Store, error) {
base, err := sqlitekit.Open(ctx, sqlitekit.Options{Path: path})
base, err := crawlstore.Open(ctx, crawlstore.Options{Path: path})
if err != nil {
return nil, err
}