feat(store): migrate historical LID rows to phone numbers

Migrate historical @lid chat/message rows to mapped phone-number JIDs after auth/session access is available. Keep FTS in sync via existing triggers and document the data repair.

Co-authored-by: Dinakar Sarbada <dinakars777@users.noreply.github.com>
This commit is contained in:
Dinakar Sarbada 2026-05-04 15:06:00 -07:00 committed by GitHub
parent d410e9f76e
commit 352caa88d8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 549 additions and 1 deletions

View File

@ -41,6 +41,7 @@
- Messages: normalize device-specific `@s.whatsapp.net` JIDs before storing chats, contacts, and senders.
- Messages: include mapped `@lid` rows when listing, searching, showing, or contextualizing by phone-number chat JID.
- Messages: read stored sender names back from SQLite and resolve blank historical `@lid` senders at display time.
- Store: migrate historical `@lid` chat and message rows to mapped phone-number JIDs during authenticated startup. (#31, #89 — thanks @bhaskoro-muthohar, @alexph-dev, and @dinakars777)
- Messages: make `messages show` prefer stored display text and include stored media/download details.
- Messages: store structured reaction target IDs and emoji in SQLite. (#67 — thanks @vlassance)
- Messages: store forwarded-message metadata and add `--forwarded` filters for list/search. (#24 — thanks @bnvyas)

View File

@ -15,6 +15,7 @@ Read when: you need the user-facing command map, global flags, store model, or l
- Write commands acquire the store lock; use `--lock-wait DURATION` to wait.
- Use `--read-only` or `WACLI_READONLY=1` to reject commands that write WhatsApp or local state.
- Use `sync --max-messages`, `sync --max-db-size`, `WACLI_SYNC_MAX_MESSAGES`, or `WACLI_SYNC_MAX_DB_SIZE` to bound local history growth.
- Authenticated startup resolves historical `@lid` chat/message rows to phone-number JIDs when the WhatsApp session store has the mapping.
## Command pages

View File

@ -128,7 +128,7 @@ func (a *App) EnsureAuthed() error {
return err
}
if a.wa.IsAuthed() {
return nil
return a.migrateHistoricalLIDs(context.Background())
}
return fmt.Errorf("not authenticated; run `wacli auth`")
}

View File

@ -0,0 +1,33 @@
package app
import (
"context"
"fmt"
"strings"
"go.mau.fi/whatsmeow/types"
)
func (a *App) migrateHistoricalLIDs(ctx context.Context) error {
if a == nil || a.db == nil || a.wa == nil {
return nil
}
lids, err := a.db.HistoricalLIDJIDs()
if err != nil {
return fmt.Errorf("load historical LID rows: %w", err)
}
for _, raw := range lids {
lid, err := types.ParseJID(strings.TrimSpace(raw))
if err != nil || lid.Server != types.HiddenUserServer {
continue
}
pn := a.wa.ResolveLIDToPN(ctx, lid)
if pn.IsEmpty() || pn.Server != types.DefaultUserServer {
continue
}
if err := a.db.MigrateLIDToPN(raw, canonicalJIDString(pn)); err != nil {
return fmt.Errorf("migrate historical LID %s: %w", raw, err)
}
}
return nil
}

View File

@ -0,0 +1,79 @@
package app
import (
"testing"
"time"
"github.com/steipete/wacli/internal/store"
"go.mau.fi/whatsmeow/types"
)
func TestEnsureAuthedMigratesHistoricalLIDs(t *testing.T) {
a := newTestApp(t)
f := newFakeWA()
a.wa = f
lid := types.JID{User: "999123456789", Device: 42, Server: types.HiddenUserServer}
lidNonAD := lid.ToNonAD()
pn := types.JID{User: "15551234567", Server: types.DefaultUserServer}
f.lids[lidNonAD] = pn
base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
if err := a.db.UpsertChat(lid.String(), "unknown", lid.String(), base); err != nil {
t.Fatalf("UpsertChat lid: %v", err)
}
if err := a.db.UpsertMessage(store.UpsertMessageParams{
ChatJID: lid.String(),
MsgID: "m-lid",
SenderJID: lid.String(),
Timestamp: base,
Text: "historical",
}); err != nil {
t.Fatalf("UpsertMessage lid: %v", err)
}
if err := a.EnsureAuthed(); err != nil {
t.Fatalf("EnsureAuthed: %v", err)
}
msg, err := a.db.GetMessage(pn.String(), "m-lid")
if err != nil {
t.Fatalf("GetMessage pn: %v", err)
}
if msg.ChatJID != pn.String() {
t.Fatalf("ChatJID = %q, want %q", msg.ChatJID, pn.String())
}
if msg.SenderJID != pn.String() {
t.Fatalf("SenderJID = %q, want %q", msg.SenderJID, pn.String())
}
lids, err := a.db.HistoricalLIDJIDs()
if err != nil {
t.Fatalf("HistoricalLIDJIDs: %v", err)
}
if len(lids) != 0 {
t.Fatalf("HistoricalLIDJIDs = %#v, want none", lids)
}
}
func TestEnsureAuthedLeavesUnresolvedHistoricalLIDs(t *testing.T) {
a := newTestApp(t)
f := newFakeWA()
a.wa = f
lid := types.JID{User: "999123456789", Server: types.HiddenUserServer}
base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
if err := a.db.UpsertChat(lid.String(), "unknown", lid.String(), base); err != nil {
t.Fatalf("UpsertChat lid: %v", err)
}
if err := a.EnsureAuthed(); err != nil {
t.Fatalf("EnsureAuthed: %v", err)
}
lids, err := a.db.HistoricalLIDJIDs()
if err != nil {
t.Fatalf("HistoricalLIDJIDs: %v", err)
}
if len(lids) != 1 || lids[0] != lid.String() {
t.Fatalf("HistoricalLIDJIDs = %#v, want %q", lids, lid.String())
}
}

View File

@ -101,6 +101,9 @@ func (a *App) Sync(ctx context.Context, opts SyncOptions) (SyncResult, error) {
return SyncResult{}, err
}
lastEvent.Store(nowUTC().UnixNano())
if err := a.migrateHistoricalLIDs(syncCtx); err != nil {
return SyncResult{MessagesStored: messagesStored.Load()}, err
}
// Optional: bootstrap imports (helps contacts/groups management without waiting for events).
if opts.RefreshContacts {

View File

@ -0,0 +1,227 @@
package store
import (
"database/sql"
"fmt"
"strings"
)
// HistoricalLIDJIDs returns distinct hidden-user JIDs stored in chat and
// message identity columns. The app layer resolves these through whatsmeow.
func (d *DB) HistoricalLIDJIDs() ([]string, error) {
rows, err := d.sql.Query(`
SELECT jid FROM chats WHERE jid GLOB '*@lid'
UNION
SELECT chat_jid FROM messages WHERE chat_jid GLOB '*@lid'
UNION
SELECT sender_jid FROM messages WHERE sender_jid GLOB '*@lid'
ORDER BY 1
`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []string
for rows.Next() {
var jid sql.NullString
if err := rows.Scan(&jid); err != nil {
return nil, err
}
if jid.Valid {
if s := strings.TrimSpace(jid.String); s != "" {
out = append(out, s)
}
}
}
return out, rows.Err()
}
// MigrateLIDToPN rewrites one historical hidden-user JID to its phone-number
// JID. It is idempotent and merges duplicate chat/message rows created by the
// old split storage behavior.
func (d *DB) MigrateLIDToPN(lidJID, pnJID string) error {
lidJID = strings.TrimSpace(lidJID)
pnJID = strings.TrimSpace(pnJID)
if lidJID == "" || pnJID == "" {
return fmt.Errorf("lid and phone-number JIDs are required")
}
if lidJID == pnJID {
return nil
}
tx, err := d.sql.Begin()
if err != nil {
return err
}
defer func() {
if tx != nil {
_ = tx.Rollback()
}
}()
if err := migrateLIDChatToPN(tx, lidJID, pnJID); err != nil {
return err
}
if err := migrateLIDMessagesToPN(tx, lidJID, pnJID); err != nil {
return err
}
if err := migrateLIDSenderToPN(tx, lidJID, pnJID); err != nil {
return err
}
if err := deleteLIDChat(tx, lidJID); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return err
}
tx = nil
return nil
}
func migrateLIDChatToPN(tx *sql.Tx, lidJID, pnJID string) error {
if _, err := tx.Exec(`
INSERT INTO chats(jid, kind, name, last_message_ts)
SELECT
?,
CASE WHEN kind = '' OR kind = 'unknown' THEN 'dm' ELSE kind END,
name,
last_message_ts
FROM chats
WHERE jid = ?
ON CONFLICT(jid) DO UPDATE SET
kind = CASE
WHEN chats.kind = '' OR chats.kind = 'unknown' OR excluded.kind = 'dm' THEN excluded.kind
ELSE chats.kind
END,
name = CASE
WHEN excluded.name IS NOT NULL
AND excluded.name != ''
AND (
chats.name IS NULL
OR chats.name = ''
OR chats.name = chats.jid
OR instr(chats.name, '@') > 0
)
THEN excluded.name
ELSE chats.name
END,
last_message_ts = max(COALESCE(chats.last_message_ts, 0), COALESCE(excluded.last_message_ts, 0))
`, pnJID, lidJID); err != nil {
return fmt.Errorf("merge lid chat into pn chat: %w", err)
}
if _, err := tx.Exec(`
INSERT INTO chats(jid, kind, name, last_message_ts)
SELECT
?,
'dm',
NULLIF(chat_name, ''),
ts
FROM messages
WHERE chat_jid = ?
ORDER BY ts DESC, rowid DESC
LIMIT 1
ON CONFLICT(jid) DO UPDATE SET
name = CASE
WHEN excluded.name IS NOT NULL
AND excluded.name != ''
AND (
chats.name IS NULL
OR chats.name = ''
OR chats.name = chats.jid
OR instr(chats.name, '@') > 0
)
THEN excluded.name
ELSE chats.name
END,
last_message_ts = max(COALESCE(chats.last_message_ts, 0), COALESCE(excluded.last_message_ts, 0))
`, pnJID, lidJID); err != nil {
return fmt.Errorf("create pn chat from lid messages: %w", err)
}
return nil
}
func migrateLIDMessagesToPN(tx *sql.Tx, lidJID, pnJID string) error {
if _, err := tx.Exec(`
INSERT INTO messages(
chat_jid, chat_name, msg_id, sender_jid, sender_name, ts, from_me, text, display_text,
is_forwarded, forwarding_score, reaction_to_id, reaction_emoji,
media_type, media_caption, filename, mime_type, direct_path,
media_key, file_sha256, file_enc_sha256, file_length, local_path, downloaded_at
)
SELECT
?,
chat_name,
msg_id,
CASE WHEN sender_jid = ? THEN ? ELSE sender_jid END,
sender_name,
ts,
from_me,
text,
display_text,
is_forwarded,
forwarding_score,
reaction_to_id,
reaction_emoji,
media_type,
media_caption,
filename,
mime_type,
direct_path,
media_key,
file_sha256,
file_enc_sha256,
file_length,
local_path,
downloaded_at
FROM messages
WHERE chat_jid = ?
ON CONFLICT(chat_jid, msg_id) DO UPDATE SET
chat_name = COALESCE(NULLIF(messages.chat_name, ''), excluded.chat_name),
sender_jid = COALESCE(NULLIF(messages.sender_jid, ''), excluded.sender_jid),
sender_name = COALESCE(NULLIF(messages.sender_name, ''), excluded.sender_name),
ts = max(messages.ts, excluded.ts),
from_me = messages.from_me,
text = COALESCE(NULLIF(messages.text, ''), excluded.text),
display_text = COALESCE(NULLIF(messages.display_text, ''), excluded.display_text),
is_forwarded = CASE WHEN messages.is_forwarded != 0 THEN messages.is_forwarded ELSE excluded.is_forwarded END,
forwarding_score = max(messages.forwarding_score, excluded.forwarding_score),
reaction_to_id = COALESCE(NULLIF(messages.reaction_to_id, ''), excluded.reaction_to_id),
reaction_emoji = COALESCE(NULLIF(messages.reaction_emoji, ''), excluded.reaction_emoji),
media_type = COALESCE(NULLIF(messages.media_type, ''), excluded.media_type),
media_caption = COALESCE(NULLIF(messages.media_caption, ''), excluded.media_caption),
filename = COALESCE(NULLIF(messages.filename, ''), excluded.filename),
mime_type = COALESCE(NULLIF(messages.mime_type, ''), excluded.mime_type),
direct_path = COALESCE(NULLIF(messages.direct_path, ''), excluded.direct_path),
media_key = CASE WHEN messages.media_key IS NOT NULL AND length(messages.media_key) > 0 THEN messages.media_key ELSE excluded.media_key END,
file_sha256 = CASE WHEN messages.file_sha256 IS NOT NULL AND length(messages.file_sha256) > 0 THEN messages.file_sha256 ELSE excluded.file_sha256 END,
file_enc_sha256 = CASE WHEN messages.file_enc_sha256 IS NOT NULL AND length(messages.file_enc_sha256) > 0 THEN messages.file_enc_sha256 ELSE excluded.file_enc_sha256 END,
file_length = CASE WHEN messages.file_length IS NOT NULL AND messages.file_length > 0 THEN messages.file_length ELSE excluded.file_length END,
local_path = COALESCE(NULLIF(messages.local_path, ''), excluded.local_path),
downloaded_at = CASE WHEN messages.downloaded_at IS NOT NULL AND messages.downloaded_at > 0 THEN messages.downloaded_at ELSE excluded.downloaded_at END
`, pnJID, lidJID, pnJID, lidJID); err != nil {
return fmt.Errorf("merge lid messages into pn chat: %w", err)
}
if _, err := tx.Exec(`DELETE FROM messages WHERE chat_jid = ?`, lidJID); err != nil {
return fmt.Errorf("delete migrated lid messages: %w", err)
}
return nil
}
func migrateLIDSenderToPN(tx *sql.Tx, lidJID, pnJID string) error {
if _, err := tx.Exec(`UPDATE messages SET sender_jid = ? WHERE sender_jid = ?`, pnJID, lidJID); err != nil {
return fmt.Errorf("rewrite lid message senders: %w", err)
}
return nil
}
func deleteLIDChat(tx *sql.Tx, lidJID string) error {
if _, err := tx.Exec(`DELETE FROM chats WHERE jid = ?`, lidJID); err != nil {
return fmt.Errorf("delete migrated lid chat: %w", err)
}
return nil
}

View File

@ -0,0 +1,166 @@
package store
import (
"reflect"
"testing"
"time"
)
func TestHistoricalLIDJIDsFindsChatAndMessageColumns(t *testing.T) {
db := openTestDB(t)
base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
pn := "15551234567@s.whatsapp.net"
lid := "999123456789@lid"
group := "120363000000@g.us"
for _, jid := range []string{pn, lid, group} {
if err := db.UpsertChat(jid, "dm", jid, base); err != nil {
t.Fatalf("UpsertChat %s: %v", jid, err)
}
}
if err := db.UpsertMessage(UpsertMessageParams{
ChatJID: lid,
MsgID: "lid-chat",
SenderJID: lid,
Timestamp: base,
Text: "lid chat",
}); err != nil {
t.Fatalf("UpsertMessage lid chat: %v", err)
}
if err := db.UpsertMessage(UpsertMessageParams{
ChatJID: group,
MsgID: "group-sender",
SenderJID: lid,
Timestamp: base,
Text: "group sender",
}); err != nil {
t.Fatalf("UpsertMessage group sender: %v", err)
}
got, err := db.HistoricalLIDJIDs()
if err != nil {
t.Fatalf("HistoricalLIDJIDs: %v", err)
}
if want := []string{lid}; !reflect.DeepEqual(got, want) {
t.Fatalf("HistoricalLIDJIDs = %#v, want %#v", got, want)
}
}
func TestMigrateLIDToPNMergesChatsAndMessages(t *testing.T) {
db := openTestDB(t)
base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
pn := "15551234567@s.whatsapp.net"
lid := "999123456789@lid"
group := "120363000000@g.us"
if err := db.UpsertChat(pn, "dm", "Alice", base); err != nil {
t.Fatalf("UpsertChat pn: %v", err)
}
if err := db.UpsertChat(lid, "unknown", lid, base.Add(10*time.Second)); err != nil {
t.Fatalf("UpsertChat lid: %v", err)
}
if err := db.UpsertChat(group, "group", "Project", base); err != nil {
t.Fatalf("UpsertChat group: %v", err)
}
if err := db.UpsertMessage(UpsertMessageParams{
ChatJID: pn,
MsgID: "dupe",
SenderJID: "",
Timestamp: base,
}); err != nil {
t.Fatalf("UpsertMessage pn dupe: %v", err)
}
if err := db.UpsertMessage(UpsertMessageParams{
ChatJID: lid,
ChatName: "Alice LID",
MsgID: "dupe",
SenderJID: lid,
SenderName: "Alice",
Timestamp: base.Add(5 * time.Second),
Text: "from lid",
}); err != nil {
t.Fatalf("UpsertMessage lid dupe: %v", err)
}
if err := db.UpsertMessage(UpsertMessageParams{
ChatJID: lid,
ChatName: "Alice LID",
MsgID: "lid-only",
SenderJID: lid,
SenderName: "Alice",
Timestamp: base.Add(6 * time.Second),
Text: "only on lid",
}); err != nil {
t.Fatalf("UpsertMessage lid only: %v", err)
}
if err := db.UpsertMessage(UpsertMessageParams{
ChatJID: group,
MsgID: "group",
SenderJID: lid,
Timestamp: base.Add(7 * time.Second),
Text: "group message",
}); err != nil {
t.Fatalf("UpsertMessage group: %v", err)
}
if err := db.MigrateLIDToPN(lid, pn); err != nil {
t.Fatalf("MigrateLIDToPN: %v", err)
}
if err := db.MigrateLIDToPN(lid, pn); err != nil {
t.Fatalf("MigrateLIDToPN idempotent: %v", err)
}
if got := countRows(t, db.sql, "SELECT COUNT(*) FROM chats WHERE jid = ?", lid); got != 0 {
t.Fatalf("lid chat rows = %d, want 0", got)
}
if got := countRows(t, db.sql, "SELECT COUNT(*) FROM messages WHERE chat_jid = ?", lid); got != 0 {
t.Fatalf("lid chat message rows = %d, want 0", got)
}
if got := countRows(t, db.sql, "SELECT COUNT(*) FROM messages WHERE sender_jid = ?", lid); got != 0 {
t.Fatalf("lid sender rows = %d, want 0", got)
}
if got := countRows(t, db.sql, "SELECT COUNT(*) FROM messages WHERE chat_jid = ?", pn); got != 2 {
t.Fatalf("pn message rows = %d, want 2", got)
}
chat, err := db.GetChat(pn)
if err != nil {
t.Fatalf("GetChat pn: %v", err)
}
if chat.Name != "Alice" {
t.Fatalf("merged chat name = %q, want Alice", chat.Name)
}
if !chat.LastMessageTS.Equal(base.Add(10 * time.Second)) {
t.Fatalf("merged chat timestamp = %s, want %s", chat.LastMessageTS, base.Add(10*time.Second))
}
dupe, err := db.GetMessage(pn, "dupe")
if err != nil {
t.Fatalf("GetMessage dupe: %v", err)
}
if dupe.Text != "from lid" {
t.Fatalf("merged duplicate text = %q, want from lid", dupe.Text)
}
if dupe.SenderJID != pn {
t.Fatalf("merged duplicate sender = %q, want %q", dupe.SenderJID, pn)
}
if !dupe.Timestamp.Equal(base.Add(5 * time.Second)) {
t.Fatalf("merged duplicate timestamp = %s, want %s", dupe.Timestamp, base.Add(5*time.Second))
}
groupMsg, err := db.GetMessage(group, "group")
if err != nil {
t.Fatalf("GetMessage group: %v", err)
}
if groupMsg.SenderJID != pn {
t.Fatalf("group sender = %q, want %q", groupMsg.SenderJID, pn)
}
lids, err := db.HistoricalLIDJIDs()
if err != nil {
t.Fatalf("HistoricalLIDJIDs after migrate: %v", err)
}
if len(lids) != 0 {
t.Fatalf("HistoricalLIDJIDs after migrate = %#v, want none", lids)
}
}

View File

@ -67,6 +67,44 @@ func TestExistingEmptyFTSTableDetectedOnReopen(t *testing.T) {
}
}
func TestMigrateLIDToPNMaintainsFTSRows(t *testing.T) {
db := openTestDB(t)
if !db.HasFTS() {
t.Skip("FTS5 not enabled")
}
base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
pn := "15551234567@s.whatsapp.net"
lid := "999123456789@lid"
if err := db.UpsertChat(lid, "unknown", lid, base); err != nil {
t.Fatalf("UpsertChat lid: %v", err)
}
if err := db.UpsertMessage(UpsertMessageParams{
ChatJID: lid,
MsgID: "m-lid",
SenderJID: lid,
Timestamp: base,
Text: "needle migrated",
}); err != nil {
t.Fatalf("UpsertMessage lid: %v", err)
}
if err := db.MigrateLIDToPN(lid, pn); err != nil {
t.Fatalf("MigrateLIDToPN: %v", err)
}
msgs, err := db.SearchMessages(SearchMessagesParams{Query: "needle", ChatJID: pn, Limit: 10})
if err != nil {
t.Fatalf("SearchMessages: %v", err)
}
if len(msgs) != 1 {
t.Fatalf("search results = %d, want 1", len(msgs))
}
if msgs[0].ChatJID != pn {
t.Fatalf("search result chat = %q, want %q", msgs[0].ChatJID, pn)
}
}
// TestSanitizeFTSQuery verifies that user input is sanitized before being
// passed to the FTS5 MATCH clause, preventing query-syntax injection (#57).
func TestSanitizeFTSQuery(t *testing.T) {