From fc40530eebc0e022bd3e39e2bf71f3932720a5b6 Mon Sep 17 00:00:00 2001
From: Vincent Koc <vincentkoc@ieee.org>
Date: Sun, 26 Apr 2026 23:29:39 -0700
Subject: [PATCH] feat: persist thread documents

---
 internal/store/documents.go      | 35 ++++++++++++++++++++++++
 internal/store/documents_test.go | 46 ++++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 internal/store/documents.go
 create mode 100644 internal/store/documents_test.go

diff --git a/internal/store/documents.go b/internal/store/documents.go
new file mode 100644
index 0000000..827cc70
--- /dev/null
+++ b/internal/store/documents.go
@@ -0,0 +1,35 @@
+package store
+
+import (
+	"context"
+	"fmt"
+)
+
+type Document struct {
+	ID         int64  `json:"id"`
+	ThreadID   int64  `json:"thread_id"`
+	Title      string `json:"title"`
+	Body       string `json:"body,omitempty"`
+	RawText    string `json:"raw_text"`
+	DedupeText string `json:"dedupe_text"`
+	UpdatedAt  string `json:"updated_at"`
+}
+
+func (s *Store) UpsertDocument(ctx context.Context, doc Document) (int64, error) {
+	var id int64
+	err := s.db.QueryRowContext(ctx, `
+		insert into documents(thread_id, title, body, raw_text, dedupe_text, updated_at)
+		values(?, ?, ?, ?, ?, ?)
+		on conflict(thread_id) do update set
+			title=excluded.title,
+			body=excluded.body,
+			raw_text=excluded.raw_text,
+			dedupe_text=excluded.dedupe_text,
+			updated_at=excluded.updated_at
+		returning id
+	`, doc.ThreadID, doc.Title, nullString(doc.Body), doc.RawText, doc.DedupeText, doc.UpdatedAt).Scan(&id)
+	if err != nil {
+		return 0, fmt.Errorf("upsert document: %w", err)
+	}
+	return id, nil
+}
diff --git a/internal/store/documents_test.go b/internal/store/documents_test.go
new file mode 100644
index 0000000..66f0142
--- /dev/null
+++ b/internal/store/documents_test.go
@@ -0,0 +1,46 @@
+package store
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+)
+
+func TestUpsertDocumentIndexesFTS(t *testing.T) {
+	ctx := context.Background()
+	st, err := Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db"))
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	defer st.Close()
+
+	repoID, err := st.UpsertRepository(ctx, Repository{Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", RawJSON: "{}", UpdatedAt: "2026-04-26T00:00:00Z"})
+	if err != nil {
+		t.Fatalf("repo: %v", err)
+	}
+	threadID, err := st.UpsertThread(ctx, Thread{
+		RepoID: repoID, GitHubID: "1", Number: 1, Kind: "issue", State: "open",
+		Title: "download stalls", HTMLURL: "https://github.com/openclaw/gitcrawl/issues/1",
+		LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "hash", UpdatedAt: "2026-04-26T00:00:00Z",
+	})
+	if err != nil {
+		t.Fatalf("thread: %v", err)
+	}
+	if _, err := st.UpsertDocument(ctx, Document{
+		ThreadID:   threadID,
+		Title:      "download stalls",
+		RawText:    "download stalls on large artifacts",
+		DedupeText: "download stalls large artifacts",
+		UpdatedAt:  "2026-04-26T00:00:00Z",
+	}); err != nil {
+		t.Fatalf("document: %v", err)
+	}
+
+	var count int
+	if err := st.DB().QueryRowContext(ctx, `select count(*) from documents_fts where documents_fts match 'artifacts'`).Scan(&count); err != nil {
+		t.Fatalf("fts query: %v", err)
+	}
+	if count != 1 {
+		t.Fatalf("fts count: got %d want 1", count)
+	}
+}