fix(search): rank results and include comments

.github/workflows/release.yml

@@ -50,7 +50,7 @@ jobs:
           echo "RELEASE_VERSION=${TAG#v}" >> "$GITHUB_ENV"
 
       - name: Build release artifacts
-        uses: goreleaser/goreleaser-action@v7
+        uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser
           version: "~> v2"

internal/share/share.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -14,7 +13,6 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
-	"syscall"
 	"time"
 
 	"github.com/vincentkoc/notcrawl/internal/store"
@@ -72,42 +70,28 @@ func Publish(ctx context.Context, st *store.Store, opts PublishOptions) (Publish
 	if err := ensureRepo(ctx, opts.RepoPath, opts.Remote, opts.Branch); err != nil {
 		return PublishSummary{}, err
 	}
-	dataRoot := filepath.Join(opts.RepoPath, "data")
-	pagesRoot := filepath.Join(opts.RepoPath, "pages")
-	if err := os.MkdirAll(dataRoot, 0o755); err != nil {
+	if err := os.RemoveAll(filepath.Join(opts.RepoPath, "data")); err != nil {
 		return PublishSummary{}, err
 	}
-	if err := os.MkdirAll(pagesRoot, 0o755); err != nil {
+	if err := os.RemoveAll(filepath.Join(opts.RepoPath, "pages")); err != nil {
+		return PublishSummary{}, err
+	}
+	if err := os.MkdirAll(filepath.Join(opts.RepoPath, "data"), 0o755); err != nil {
 		return PublishSummary{}, err
 	}
 	manifest := Manifest{GeneratedAt: time.Now().UTC().Format(time.RFC3339)}
-	dataKeep := map[string]bool{}
 	for _, table := range exportTables {
 		tm, err := exportTable(ctx, st.DB(), opts.RepoPath, table)
 		if err != nil {
 			return PublishSummary{}, err
 		}
 		manifest.Tables = append(manifest.Tables, tm)
-		dataKeep[filepath.Clean(filepath.Join(opts.RepoPath, tm.Path))] = true
 	}
-	if err := pruneGeneratedFiles(dataRoot, dataKeep, func(path string) bool {
-		return strings.HasSuffix(path, ".jsonl.gz")
-	}); err != nil {
-		return PublishSummary{}, err
-	}
-	pagesKeep := map[string]bool{}
 	if opts.MarkdownDir != "" {
-		var err error
-		pagesKeep, err = copyDir(opts.MarkdownDir, pagesRoot)
-		if err != nil && !os.IsNotExist(err) {
+		if err := copyDir(opts.MarkdownDir, filepath.Join(opts.RepoPath, "pages")); err != nil && !os.IsNotExist(err) {
 			return PublishSummary{}, err
 		}
 	}
-	if err := pruneGeneratedFiles(pagesRoot, pagesKeep, func(path string) bool {
-		return strings.HasSuffix(path, ".md")
-	}); err != nil {
-		return PublishSummary{}, err
-	}
 	b, err := json.MarshalIndent(manifest, "", "  ")
 	if err != nil {
 		return PublishSummary{}, err
@@ -333,16 +317,15 @@ func run(ctx context.Context, dir, name string, args ...string) error {
 	return nil
 }
 
-func copyDir(src, dst string) (map[string]bool, error) {
+func copyDir(src, dst string) error {
 	info, err := os.Stat(src)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	if !info.IsDir() {
-		return nil, fmt.Errorf("not a directory: %s", src)
+		return fmt.Errorf("not a directory: %s", src)
 	}
-	keep := map[string]bool{}
-	err = filepath.WalkDir(src, func(path string, d os.DirEntry, err error) error {
+	return filepath.WalkDir(src, func(path string, d os.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
@@ -367,51 +350,9 @@ func copyDir(src, dst string) (map[string]bool, error) {
 			return err
 		}
 		defer out.Close()
-		if _, err := io.Copy(out, in); err != nil {
-			return err
-		}
-		keep[filepath.Clean(target)] = true
-		return nil
-	})
-	return keep, err
-}
-
-func pruneGeneratedFiles(root string, keep map[string]bool, shouldPrune func(string) bool) error {
-	if _, err := os.Stat(root); err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
+		_, err = io.Copy(out, in)
 		return err
-	}
-	var dirs []string
-	if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-		if path == root {
-			return nil
-		}
-		if d.IsDir() {
-			dirs = append(dirs, path)
-			return nil
-		}
-		clean := filepath.Clean(path)
-		if shouldPrune(clean) && !keep[clean] {
-			return os.Remove(clean)
-		}
-		return nil
-	}); err != nil {
-		return err
-	}
-	sort.Slice(dirs, func(i, j int) bool {
-		return len(dirs[i]) > len(dirs[j])
 	})
-	for _, dir := range dirs {
-		if err := os.Remove(dir); err != nil && !os.IsNotExist(err) && !errors.Is(err, syscall.ENOTEMPTY) && !errors.Is(err, syscall.EEXIST) {
-			return err
-		}
-	}
-	return nil
 }
 
 func exportValue(v any) any {

internal/share/share_test.go

@@ -39,35 +39,6 @@ func TestPublishAndImportSnapshot(t *testing.T) {
 	if _, err := os.Stat(filepath.Join(repo, "pages", "default", "launch-page1.md")); err != nil {
 		t.Fatal(err)
 	}
-	stalePage := filepath.Join(repo, "pages", "default", "stale.md")
-	if err := os.WriteFile(stalePage, []byte("stale"), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	pageSidecar := filepath.Join(repo, "pages", "default", "README.txt")
-	if err := os.WriteFile(pageSidecar, []byte("keep"), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	staleData := filepath.Join(repo, "data", "stale.jsonl.gz")
-	if err := os.WriteFile(staleData, []byte("stale"), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	dataSidecar := filepath.Join(repo, "data", "README.txt")
-	if err := os.WriteFile(dataSidecar, []byte("keep"), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := Publish(ctx, src, PublishOptions{RepoPath: repo, MarkdownDir: mdDir}); err != nil {
-		t.Fatal(err)
-	}
-	for _, path := range []string{stalePage, staleData} {
-		if _, err := os.Stat(path); !os.IsNotExist(err) {
-			t.Fatalf("expected generated stale file %s to be pruned, got %v", path, err)
-		}
-	}
-	for _, path := range []string{pageSidecar, dataSidecar} {
-		if _, err := os.Stat(path); err != nil {
-			t.Fatalf("expected sidecar %s to remain: %v", path, err)
-		}
-	}
 	dst, err := store.Open(filepath.Join(t.TempDir(), "dst.db"))
 	if err != nil {
 		t.Fatal(err)