fix(markdown): prune stale exported files
Some checks failed
Validation / validate (push) Has been cancelled
Some checks failed
Validation / validate (push) Has been cancelled
This commit is contained in:
parent
a906ed4845
commit
5ca3e4f38f
3
SPEC.md
3
SPEC.md
@ -112,6 +112,9 @@ separators and unsafe punctuation with dashes:
|
||||
pages/<space-slug>/<page-title>-<short-id>.md
|
||||
```
|
||||
|
||||
Each export removes stale generated `.md` files under the Markdown root while
|
||||
leaving non-Markdown sidecar files alone.
|
||||
|
||||
Each file starts with YAML-ish front matter:
|
||||
|
||||
```yaml
|
||||
|
||||
@ -2,11 +2,13 @@ package markdown
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/vincentkoc/notcrawl/internal/notiontext"
|
||||
@ -38,14 +40,19 @@ func (e Exporter) Export(ctx context.Context) (Summary, error) {
|
||||
return Summary{}, err
|
||||
}
|
||||
var s Summary
|
||||
keep := map[string]bool{}
|
||||
for _, page := range pages {
|
||||
path, err := e.writePage(ctx, page)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
keep[filepath.Clean(path)] = true
|
||||
s.Pages++
|
||||
s.Files = append(s.Files, path)
|
||||
}
|
||||
if err := pruneStaleMarkdown(e.Dir, keep); err != nil {
|
||||
return s, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
@ -211,6 +218,41 @@ func fallback(s, fallback string) string {
|
||||
return fallback
|
||||
}
|
||||
|
||||
func pruneStaleMarkdown(root string, keep map[string]bool) error {
|
||||
var dirs []string
|
||||
if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path = filepath.Clean(path)
|
||||
if d.IsDir() {
|
||||
if path != filepath.Clean(root) {
|
||||
dirs = append(dirs, path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if filepath.Ext(path) == ".md" && !keep[path] {
|
||||
return os.Remove(path)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(dirs, func(i, j int) bool {
|
||||
return len(dirs[i]) > len(dirs[j])
|
||||
})
|
||||
for _, dir := range dirs {
|
||||
if err := os.Remove(dir); err != nil && !isIgnorableRemoveDirError(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isIgnorableRemoveDirError(err error) bool {
|
||||
return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTEMPTY) || errors.Is(err, syscall.EEXIST)
|
||||
}
|
||||
|
||||
func formatMS(ms int64) string {
|
||||
if ms <= 0 {
|
||||
return ""
|
||||
|
||||
@ -104,3 +104,40 @@ func TestExporterPreservesUnicodePathNames(t *testing.T) {
|
||||
t.Fatalf("unexpected export path: %+v, want %s", s.Files, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExporterPrunesStaleMarkdown(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st, err := store.Open(filepath.Join(t.TempDir(), "notcrawl.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Close()
|
||||
now := store.NowMS()
|
||||
if err := st.UpsertPage(ctx, store.Page{ID: "page1", Title: "Launch", Alive: true, Source: "test", SyncedAt: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
staleDir := filepath.Join(dir, "old")
|
||||
if err := os.MkdirAll(staleDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
staleMarkdown := filepath.Join(staleDir, "stale.md")
|
||||
if err := os.WriteFile(staleMarkdown, []byte("old"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keepNote := filepath.Join(staleDir, "note.txt")
|
||||
if err := os.WriteFile(keepNote, []byte("keep"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := (Exporter{Store: st, Dir: dir}).Export(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(staleMarkdown); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected stale markdown to be removed, stat err=%v", err)
|
||||
}
|
||||
if _, err := os.Stat(keepNote); err != nil {
|
||||
t.Fatalf("expected non-markdown file to remain: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user