fix(api): skip inaccessible comments during sync

Parse Notion API errors and ignore restricted comment access so API sync can continue when comments are unavailable.

Fixes #6
This commit is contained in:
davelutztx 2026-04-27 12:49:52 -05:00 committed by GitHub
parent 45d4cc5aa2
commit 47e96fd4c0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 66 additions and 2 deletions

View File

@ -398,7 +398,7 @@ func (c Client) ingestComments(ctx context.Context, st *store.Store, pageID, spa
}
var resp obj
if err := c.do(ctx, http.MethodGet, path, nil, &resp); err != nil {
if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "not_found") {
if isIgnoredCommentError(err) {
return count, nil
}
return count, err
@ -476,11 +476,49 @@ func (c Client) do(ctx context.Context, method, path string, body any, out any)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
return fmt.Errorf("notion api %s %s: %s: %s", method, path, resp.Status, strings.TrimSpace(string(b)))
bodyText := strings.TrimSpace(string(b))
apiErr := notionAPIError{Method: method, Path: path, Status: resp.Status, StatusCode: resp.StatusCode, Body: bodyText}
var payload struct {
Code string `json:"code"`
Message string `json:"message"`
}
if err := json.Unmarshal(b, &payload); err == nil {
apiErr.Code = payload.Code
apiErr.Message = payload.Message
}
return apiErr
}
return json.NewDecoder(resp.Body).Decode(out)
}
type notionAPIError struct {
Method string
Path string
Status string
StatusCode int
Code string
Message string
Body string
}
func (e notionAPIError) Error() string {
if e.Code != "" || e.Message != "" {
return fmt.Sprintf("notion api %s %s: %s: %s: %s", e.Method, e.Path, e.Status, e.Code, e.Message)
}
return fmt.Sprintf("notion api %s %s: %s: %s", e.Method, e.Path, e.Status, e.Body)
}
func isIgnoredCommentError(err error) bool {
apiErr, ok := err.(notionAPIError)
if !ok {
return false
}
if apiErr.StatusCode == http.StatusNotFound || apiErr.Code == "not_found" {
return true
}
return apiErr.StatusCode == http.StatusForbidden && apiErr.Code == "restricted_resource"
}
func userName(u obj) string {
if name := u.string("name"); name != "" {
return name

View File

@ -187,3 +187,29 @@ func TestSyncIngestsCurrentDataSourcesAndRows(t *testing.T) {
t.Fatalf("unexpected rows: %+v", rows)
}
}
func TestIngestCommentsSkipsRestrictedResource(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path != "/comments" {
t.Fatalf("unexpected request: %s %s", r.Method, r.URL.String())
}
w.WriteHeader(http.StatusForbidden)
_, _ = w.Write([]byte(`{"object":"error","status":403,"code":"restricted_resource","message":"Insufficient permissions for this endpoint."}`))
}))
defer server.Close()
st, err := store.Open(filepath.Join(t.TempDir(), "notcrawl.db"))
if err != nil {
t.Fatal(err)
}
defer st.Close()
count, err := (Client{BaseURL: server.URL, Version: "2022-06-28", Token: "secret", HTTP: http.DefaultClient}).ingestComments(context.Background(), st, "page1", "")
if err != nil {
t.Fatal(err)
}
if count != 0 {
t.Fatalf("unexpected comment count: %d", count)
}
}