diff --git a/CHANGELOG.md b/CHANGELOG.md index dad9c54..5d13fa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## 0.3.0 - Unreleased +- Improve `gh` shim cache coordination and observability with stale-while-revalidate reads, finer Actions/API TTLs, recent-window stats, top miss keys, and `xcache snapshot`. + ## 0.2.0 - 2026-05-05 - Add Homebrew tap installation via `brew install openclaw/tap/gitcrawl`. diff --git a/README.md b/README.md index 3b40bd6..a38c448 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ gitcrawl tui owner/repo Pass `--numbers` to refresh exact issue or pull request rows without relying on list ordering or updated-time windows. Pass `--with pr-details` or `--include-pr-details` to hydrate pull request files, commits, checks, and workflow runs for local review. The `gh` shim can also auto-hydrate one exact PR on a PR-detail miss, then retry locally. `gitcrawl search issues|prs` accepts the common `gh search` shape (` -R owner/repo --state open --json fields --limit N`) and answers from the local SQLite cache. It is intended for discovery without spending GitHub REST search quota; use `gh` for final live verification and GitHub write actions. Pass `--sync-if-stale 5m` to perform one metadata sync before the cached search when the local repository mirror is older than that duration. -`gitcrawl gh` is a gh-compatible shim for agent workflows. It answers broad `gh search issues|prs`, `gh issue/pr list`, supported `gh issue/pr view --json` fields, hydrated `gh pr checks`, and hydrated `gh run list/view` from local SQLite, then falls through to the real GitHub CLI for unsupported commands. Local `gh issue/pr list` supports common filters such as `--author`, `--assignee`, and repeated `--label`. Read-only fallthroughs such as `gh pr diff`, `gh repo view/list`, `gh release list/view`, `gh workflow list/view`, `gh secret list`, `gh variable get/list`, `gh label list`, read-only `gh search` kinds, GET-only REST `gh api` calls, and read-only `gh api graphql` queries use a command-aware persistent cache under `cache/gh-shim`; Actions run/job logs get longer TTLs, completed run views are kept much longer than active CI status, user profile reads get a 7-day TTL, read-only GraphQL gets a 6-hour TTL, and `gh pr diff` entries are keyed by the cached PR head SHA when available. Explicit API paths and explicit repositories share cache entries across sibling checkouts even when agents set different `GH_REPO` values; implicit repo reads stay isolated by `GH_REPO` or current working directory. Cache keys canonicalize common flags such as `-R`/`--repo` and sorted `--json` fields so equivalent agent commands coalesce. Repeat read failures are cached by default so agents do not rediscover the same missing release or workflow, but rate-limit error entries expire quickly; if GitHub rate-limits a refresh and an expired successful entry exists, the shim serves the stale response with a warning instead of failing the read. Set `GITCRAWL_GH_CACHE_ERRORS=0` to disable error caching. Mutating commands pass through, increment write counters, and invalidate matching cache tags instead of flushing unrelated entries. `gh xcache stats|keys|gc|flush|reset` inspects, garbage-collects, clears, or resets fallthrough-cache counters, including hit rate plus per-command and per-route backend miss counters. Set `GITCRAWL_GH_PATH` to choose the backend `gh`, and symlink or install the binary as `gh`/`gitcrawl-gh` to run the shim directly. +`gitcrawl gh` is a gh-compatible shim for agent workflows. It answers broad `gh search issues|prs`, `gh issue/pr list`, supported `gh issue/pr view --json` fields, hydrated `gh pr checks`, and hydrated `gh run list/view` from local SQLite, then falls through to the real GitHub CLI for unsupported commands. Local `gh issue/pr list` supports common filters such as `--author`, `--assignee`, and repeated `--label`. Read-only fallthroughs such as `gh pr diff`, `gh repo view/list`, `gh release list/view`, `gh workflow list/view`, `gh secret list`, `gh variable get/list`, `gh label list`, read-only `gh search` kinds, GET-only REST `gh api` calls, and read-only `gh api graphql` queries use a command-aware persistent cache under `cache/gh-shim`; Actions run/job logs get longer TTLs, completed run/job reads are kept much longer than active CI status, user profile reads get a 7-day TTL, read-only GraphQL gets a 6-hour TTL, and `gh pr diff` entries are keyed by the cached PR head SHA when available. Explicit API paths and explicit repositories share cache entries across sibling checkouts even when agents set different `GH_REPO` values; implicit repo reads stay isolated by `GH_REPO` or current working directory. Cache keys canonicalize common flags such as `-R`/`--repo` and sorted `--json` fields so equivalent agent commands coalesce. Repeat read failures are cached by default so agents do not rediscover the same missing release or workflow, but rate-limit error entries expire quickly; if GitHub rate-limits a refresh and an expired successful entry exists, the shim serves the stale response with a warning instead of failing the read. When another process is refreshing an expired successful entry, peers may serve stale inside a short grace window instead of joining the backend stampede. Set `GITCRAWL_GH_STALE_GRACE=0` to disable stale-while-revalidate, or `GITCRAWL_GH_CACHE_ERRORS=0` to disable error caching. Mutating commands pass through, increment write counters, and invalidate matching cache tags instead of flushing unrelated entries. `gh xcache stats|keys|gc|flush|reset|snapshot` inspects, garbage-collects, clears, resets, or snapshots fallthrough-cache counters, including hit rate plus per-command, per-route, per-key, and `--since` recent-window miss counters. Set `GITCRAWL_GH_PATH` to choose the backend `gh`, and symlink or install the binary as `gh`/`gitcrawl-gh` to run the shim directly. The TUI starts at `--min-size 5` and `--sort size`, like ghcrawl's saved default, so the first screen is the useful cluster workload instead of singleton noise. Pass `--min-size 1` when you intentionally want singleton clusters. Mouse support is built in: click rows, wheel panes, and right-click for copy, sort, filter, jump, link, neighbor, local close/reopen, and member triage actions. Press `a` to open the same action menu from the keyboard, `#` to jump directly to an issue or PR number, `p` to switch between repositories already present in the local store, or `n` to load neighbors for the selected issue or PR. Enter from the members pane also loads neighbors before opening detail. The TUI quietly refreshes from the local store every 15 seconds. ## Local Defaults diff --git a/SPEC.md b/SPEC.md index 33f161c..643f852 100644 --- a/SPEC.md +++ b/SPEC.md @@ -122,9 +122,10 @@ gitcrawl gh xcache stats gitcrawl gh xcache keys gitcrawl gh xcache reset gitcrawl gh xcache flush +gitcrawl gh xcache snapshot [--reset] ``` -The cache key includes the resolved gitcrawl config path, current working directory, `GH_HOST`, `GH_REPO`, stable PR-diff identity when available, and canonicalized `gh` arguments. This keeps sibling checkouts and portable stores isolated while still coalescing equivalent agent calls such as reordered flags or sorted `--json` fields. Concurrent cache misses use a lock file so one process populates the entry while peers wait for the result. +The cache key includes the resolved gitcrawl config path, current working directory, `GH_HOST`, `GH_REPO`, stable PR-diff identity when available, and canonicalized `gh` arguments. This keeps sibling checkouts and portable stores isolated while still coalescing equivalent agent calls such as reordered flags or sorted `--json` fields. Concurrent cache misses use a lock file so one process populates the entry while peers wait for the result; if an expired successful entry is still inside its stale grace window, peers may serve stale while the lock holder refreshes it. `xcache stats --since ` reports recent-window counters from hourly buckets, and miss maps include command, normalized route, and canonical key views. ## Config diff --git a/docs/automation.md b/docs/automation.md index 7aa0626..3c9c3aa 100644 --- a/docs/automation.md +++ b/docs/automation.md @@ -122,6 +122,10 @@ gh issue comment 456 -R owner/repo --body "Duplicate of #123" # Periodically log cache stats — watch local_hits climb relative to backend_misses. gitcrawl gh xcache stats --json \ | jq '{local: .counters.local_hits, fallback: .counters.fallback_hits, github: .counters.backend_misses}' + +# During release/debug sessions, compare a recent window or snapshot before reset. +gitcrawl gh xcache stats --since 1h --json +gitcrawl gh xcache snapshot --reset --json ``` ## Multi-repo automation diff --git a/docs/commands.md b/docs/commands.md index af73058..f5221df 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -101,7 +101,7 @@ These work on every command. | `gitcrawl gh label list ...` | Falls through; cached briefly | [gh shim](/gh-shim/) | | `gitcrawl gh api ` | Falls through; cached briefly (GET-only REST) | [gh shim](/gh-shim/) | | `gitcrawl gh api graphql -f query=...` | Falls through; read-only queries are cached | [gh shim](/gh-shim/#read-only-fallthroughs-cached) | -| `gitcrawl gh xcache stats\|keys\|gc\|flush\|reset [--json]` | Cache inspection / housekeeping | [gh shim](/gh-shim/#cache-inspection-xcache) | +| `gitcrawl gh xcache stats [--since ] \| keys \| gc \| flush \| reset \| snapshot [--reset] [--json]` | Cache inspection / housekeeping | [gh shim](/gh-shim/#cache-inspection-xcache) | | _Anything else_ | Falls through to real `gh` | [gh shim](/gh-shim/) | The shim binary can be installed standalone by symlinking the `gitcrawl` binary as `gh` or `gitcrawl-gh`. diff --git a/docs/configuration.md b/docs/configuration.md index 2dd7113..2c261bf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -102,6 +102,7 @@ checkout_dir = "/Users/me/.config/gitcrawl/portable" | `GITCRAWL_GH_PATH` | Path to the real `gh` binary used for fallthrough | | `GITCRAWL_GH_AUTO_HYDRATE` | Set to `0` to disable PR auto-hydration on cache miss | | `GITCRAWL_GH_CACHE_TTL` | Override fallthrough cache TTL (e.g., `5m`, `1h`) | +| `GITCRAWL_GH_STALE_GRACE` | Override stale-while-revalidate grace for expired successful fallthrough entries | | `GITCRAWL_GH_CACHE_ERRORS` | Set to `0` to avoid caching non-zero read-only fallthroughs | If `GITCRAWL_GH_PATH` is unset, the shim probes common Homebrew install paths and then your `PATH`. Set it explicitly when you symlink the gitcrawl binary as `gh` (otherwise the shim will recurse into itself). diff --git a/docs/gh-shim.md b/docs/gh-shim.md index ea0e26a..2a28b21 100644 --- a/docs/gh-shim.md +++ b/docs/gh-shim.md @@ -101,9 +101,9 @@ These commands always run real `gh` but the response body is cached for the next Common Actions REST reads such as run status, job lists, and logs get Actions-aware TTLs. -Default cache TTLs are command-aware: active `gh run list` and run-status reads use `2m`; completed run views are kept for `12h`; completed run lists are kept for `30m`; workflow, job detail, and Actions job-list reads use `5m`; search reads use `15m`; release metadata uses `30m`; GitHub user profile reads use `7d`; read-only GraphQL queries use `6h`; completed-style run/job log reads use `12h`; `gh pr diff` uses `5m` without a stable SHA and `7d` with one. Most other read-only fallthroughs use `5m` to `10m`. Override with `GITCRAWL_GH_CACHE_TTL=5m` or similar. +Default cache TTLs are command-aware: active `gh run list` and run-status reads use `30s`; completed run views, completed Actions job lists, and run/job logs are kept for `12h`; completed run lists are kept for `30m`; workflow reads use `15m`; search reads use `15m`; release metadata uses `1h`; GitHub user profile reads use `7d`; read-only GraphQL queries use `6h`; GitHub Pages metadata uses `15m` to `30m`; tagged/SHA `contents` API reads use `7d`; `gh pr diff` uses `5m` without a stable SHA and `7d` with one. Most other read-only fallthroughs use `5m` to `10m`. Override with `GITCRAWL_GH_CACHE_TTL=5m` or similar. -Repeat read failures are cached by default too. That avoids a fleet of agents all rediscovering the same missing release, workflow, secret, or unsupported field. Error entries are capped to shorter lifetimes, and rate-limit errors are capped at `2m` so a reset is not masked all day. If GitHub returns a rate-limit error while refreshing an expired successful entry, the shim serves that stale success with a warning instead of failing the read. Set `GITCRAWL_GH_CACHE_ERRORS=0` to cache successful reads only. +Repeat read failures are cached by default too. That avoids a fleet of agents all rediscovering the same missing release, workflow, secret, or unsupported field. Error entries are capped to shorter lifetimes, and rate-limit errors are capped at `2m` so a reset is not masked all day. If GitHub returns a rate-limit error while refreshing an expired successful entry, the shim serves that stale success with a warning instead of failing the read. When another process is already refreshing an expired successful entry, peers can serve that stale entry within a short command-aware grace window instead of joining the backend stampede. Set `GITCRAWL_GH_STALE_GRACE=0` to disable stale-while-revalidate, or `GITCRAWL_GH_CACHE_ERRORS=0` to cache successful reads only. ## Auto-hydration @@ -124,9 +124,10 @@ gitcrawl gh xcache keys # per-entry detail gitcrawl gh xcache gc # remove expired entries + stale lock files gitcrawl gh xcache flush # clear everything gitcrawl gh xcache reset # reset counters without deleting entries +gitcrawl gh xcache snapshot # write a counter snapshot for later comparison ``` -All accept `--json` for scripting. +All accept `--json` for scripting. `stats` accepts `--since 1h` for recent-window counters. `snapshot` accepts `--reset` to checkpoint counters before a noisy release/debugging session. `stats` JSON: @@ -152,6 +153,9 @@ All accept `--json` for scripting. }, "backend_misses_by_route": { "api repos/:owner/:repo/actions/runs/:id/logs": 3 + }, + "backend_misses_by_key": { + "api repos/openclaw/gitcrawl/actions/runs/123/logs -i": 2 } }, "commands": { @@ -161,7 +165,7 @@ All accept `--json` for scripting. } ``` -`local_hits` are answered from SQLite; `fallback_hits` are answered from the fallthrough cache; `stale_hits` are expired successful cache entries served after a backend rate-limit response; `backend_misses` actually hit GitHub. The per-command and per-route miss maps show which shapes still escape the cache, which is usually the fastest way to find the next optimization. +`local_hits` are answered from SQLite; `fallback_hits` are answered from the fallthrough cache; `stale_hits` are expired successful cache entries served after a backend rate-limit response or while another process refreshes the key; `backend_misses` actually hit GitHub. The per-command, per-route, and per-key miss maps show which shapes still escape the cache, which is usually the fastest way to find the next optimization. ## Cache key composition diff --git a/docs/reference.md b/docs/reference.md index 958b412..7700eb3 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -104,17 +104,21 @@ Override the config root with `--config ` or `GITCRAWL_CONFIG`. | Cache class | TTL | | --- | --- | | Most read-only fallthroughs | `5m`-`10m` | -| `gh run list` / run status | `2m` | +| `gh run list` / run status | `30s` | | `gh run view --log` / `--log-failed` | `12h` | -| `gh run view --job` | `5m` | +| `gh run view --job` | `1m` | | `gh search ...` | `15m` | -| `gh release ...` | `30m` | -| `gh api` Actions run status | `2m` | -| `gh api` Actions job lists / workflow reads | `5m` | +| `gh release ...` | `1h` | +| `gh api` Actions run status | `30s` | +| `gh api` Actions job lists | `1m` active, `12h` completed | +| `gh api` workflow reads | `15m` | | `gh api` Actions run/job logs | `12h` | +| `gh api` Pages metadata | `15m`-`30m` | +| `gh api` tagged/SHA contents | `7d` | | `gh pr diff` without stable head SHA | `5m` | | `gh pr diff` with stable head SHA | `7d` | | Override | `GITCRAWL_GH_CACHE_TTL` | +| Stale-while-revalidate grace | command-aware; override with `GITCRAWL_GH_STALE_GRACE` | | Cache read failures | on by default; error TTL is capped (`2m` for rate-limit errors); disable with `GITCRAWL_GH_CACHE_ERRORS=0` | ## gh shim cache key composition diff --git a/internal/cli/gh_shim_cache.go b/internal/cli/gh_shim_cache.go index c1a4f08..7142236 100644 --- a/internal/cli/gh_shim_cache.go +++ b/internal/cli/gh_shim_cache.go @@ -41,8 +41,8 @@ func (a *App) execRealGHMaybeCached(ctx context.Context, args []string) error { lockPath := entryPath + ".lock" lock, locked := tryGHCommandCacheLock(lockPath) if !locked { - if entry, ok := waitGHCommandCache(entryPath, lockPath, ttl); ok { - _ = a.incrementGHXCacheCounter("fallback_hits") + if entry, hit, ok := waitGHCommandCache(entryPath, lockPath, ttl, staleEntry, hasStaleEntry); ok { + _ = a.incrementGHXCacheCounter(hit) return a.writeGHCommandCacheEntry(entry) } lock, locked = tryGHCommandCacheLock(lockPath) @@ -60,7 +60,7 @@ func (a *App) execRealGHMaybeCached(ctx context.Context, args []string) error { stdout, stderr, exitCode, err := a.captureRealGH(ctx, args) _ = a.incrementGHXCacheBackendMiss(args) - if err != nil && hasStaleEntry && staleEntry.ExitCode == 0 && ghCommandOutputLooksRateLimited(stdout, stderr) { + if err != nil && hasStaleEntry && ghCommandCacheEntryCanServeStale(staleEntry, ttl) && ghCommandOutputLooksRateLimited(stdout, stderr) { _ = a.incrementGHXCacheCounter("stale_hits") _, _ = fmt.Fprintf(a.Stderr, "gitcrawl: GitHub rate limited; serving stale cached gh response from %s ago\n", time.Since(staleEntry.CreatedAt).Round(time.Second)) return a.writeGHCommandCacheEntry(staleEntry) @@ -216,6 +216,47 @@ func ghCommandCacheEntryTTL(entry ghCommandCacheEntry, ttl time.Duration) time.D return ttl } +func ghCommandCacheEntryCanServeStale(entry ghCommandCacheEntry, ttl time.Duration) bool { + if entry.ExitCode != 0 || entry.CreatedAt.IsZero() { + return false + } + age := time.Since(entry.CreatedAt) + if age <= ghCommandCacheEntryTTL(entry, ttl) { + return true + } + return age <= ghCommandCacheEntryTTL(entry, ttl)+ghCommandCacheStaleGrace(entry.Args) +} + +func ghCommandCacheStaleGrace(args []string) time.Duration { + if raw := strings.TrimSpace(os.Getenv("GITCRAWL_GH_STALE_GRACE")); raw != "" { + if duration, err := time.ParseDuration(raw); err == nil && duration >= 0 { + return duration + } + } + if len(args) == 0 { + return 5 * time.Minute + } + switch args[0] { + case "run": + return 2 * time.Minute + case "api": + route := normalizeGHAPIRoute(args[1:]) + switch { + case strings.Contains(route, "/actions/runs"): + return 2 * time.Minute + case strings.Contains(route, "/pages"): + return 30 * time.Minute + case strings.Contains(route, "/contents"): + return 6 * time.Hour + case strings.HasPrefix(route, "api users/"): + return 24 * time.Hour + } + case "release", "workflow", "repo": + return 30 * time.Minute + } + return 10 * time.Minute +} + func ghCommandCacheEntryLooksRateLimited(entry ghCommandCacheEntry) bool { return ghCommandOutputLooksRateLimited(entry.Stdout, entry.Stderr) } @@ -270,19 +311,28 @@ func tryGHCommandCacheLock(path string) (*os.File, bool) { return lock, true } -func waitGHCommandCache(entryPath, lockPath string, ttl time.Duration) (ghCommandCacheEntry, bool) { +func waitGHCommandCache(entryPath, lockPath string, ttl time.Duration, staleEntry ghCommandCacheEntry, hasStaleEntry bool) (ghCommandCacheEntry, string, bool) { + if hasStaleEntry && ghCommandCacheEntryCanServeStale(staleEntry, ttl) { + time.Sleep(250 * time.Millisecond) + if entry, ok := readGHCommandCache(entryPath, ttl); ok { + return entry, "fallback_hits", true + } + if _, err := os.Stat(lockPath); err == nil { + return staleEntry, "stale_hits", true + } + } deadline := time.Now().Add(30 * time.Second) for time.Now().Before(deadline) { time.Sleep(100 * time.Millisecond) if entry, ok := readGHCommandCache(entryPath, ttl); ok { - return entry, true + return entry, "fallback_hits", true } if _, err := os.Stat(lockPath); os.IsNotExist(err) { - return ghCommandCacheEntry{}, false + return ghCommandCacheEntry{}, "", false } } _ = os.Remove(lockPath) - return ghCommandCacheEntry{}, false + return ghCommandCacheEntry{}, "", false } func (a *App) ghCommandCacheKey(ctx context.Context, args []string) string { diff --git a/internal/cli/gh_shim_cache_policy.go b/internal/cli/gh_shim_cache_policy.go index 0e4979e..e3e6e9f 100644 --- a/internal/cli/gh_shim_cache_policy.go +++ b/internal/cli/gh_shim_cache_policy.go @@ -3,6 +3,7 @@ package cli import ( "context" "encoding/json" + "net/url" "os" "strings" "time" @@ -189,7 +190,7 @@ func ghCommandCacheTTLBase(args []string, stablePRDiff bool) time.Duration { func ghRunCacheTTL(args []string) time.Duration { if len(args) == 0 { - return 2 * time.Minute + return 30 * time.Second } switch args[0] { case "view": @@ -197,13 +198,13 @@ func ghRunCacheTTL(args []string) time.Duration { return 12 * time.Hour } if hasAnyGHFlag(args[1:], "--job") { - return 5 * time.Minute + return 1 * time.Minute } - return 2 * time.Minute + return 30 * time.Second case "list": - return 2 * time.Minute + return 30 * time.Second default: - return 2 * time.Minute + return 30 * time.Second } } @@ -214,20 +215,35 @@ func ghAPICacheTTL(args []string) time.Duration { return 6 * time.Hour case strings.HasPrefix(route, "api users/"): return 7 * 24 * time.Hour + case strings.Contains(route, "/contents"): + if ghAPIContentRefIsStable(args) { + return 7 * 24 * time.Hour + } + return 30 * time.Minute + case strings.Contains(route, "/pages/builds/latest"): + return 2 * time.Minute + case strings.Contains(route, "/pages/health"): + return 15 * time.Minute + case strings.Contains(route, "/pages"): + return 30 * time.Minute case strings.Contains(route, "/actions/runs/:id/logs"): return 12 * time.Hour case strings.Contains(route, "/actions/jobs/:id/logs"): return 12 * time.Hour case strings.Contains(route, "/actions/runs/:id/jobs"): - return 5 * time.Minute + return 1 * time.Minute + case strings.Contains(route, "/actions/jobs/:id"): + return 1 * time.Minute + case strings.Contains(route, "/pending_deployments"): + return 30 * time.Second case strings.Contains(route, "/actions/runs/:id"): - return 2 * time.Minute + return 30 * time.Second case strings.Contains(route, "/actions/workflows/"): - return 5 * time.Minute + return 15 * time.Minute case strings.Contains(route, "/actions/runs"): - return 2 * time.Minute + return 30 * time.Second case strings.Contains(route, "/releases"): - return 30 * time.Minute + return 1 * time.Hour case strings.Contains(route, "/branches") || strings.Contains(route, "/commits"): return 10 * time.Minute default: @@ -235,6 +251,62 @@ func ghAPICacheTTL(args []string) time.Duration { } } +func ghAPIContentRefIsStable(args []string) bool { + path := ghAPIPathArg(args) + _, rawQuery, found := strings.Cut(path, "?") + if !found { + return false + } + for _, part := range strings.Split(rawQuery, "&") { + name, value, ok := strings.Cut(part, "=") + if !ok || name != "ref" { + continue + } + value = strings.TrimSpace(value) + if decoded, err := url.QueryUnescape(value); err == nil { + value = strings.TrimSpace(decoded) + } + if len(value) == 40 && isHexString(value) { + return true + } + if ghAPIContentRefIsStableReleaseTag(value) { + return true + } + } + return false +} + +func ghAPIContentRefIsStableReleaseTag(value string) bool { + value = strings.TrimSpace(value) + if strings.HasPrefix(value, "refs/heads/") { + return false + } + value = strings.TrimPrefix(value, "refs/tags/") + if strings.HasPrefix(value, "refs/") { + return false + } + if strings.HasPrefix(value, "v") { + value = strings.TrimPrefix(value, "v") + } + core := value + if before, _, found := strings.Cut(core, "+"); found { + core = before + } + if before, _, found := strings.Cut(core, "-"); found { + core = before + } + parts := strings.Split(core, ".") + if len(parts) != 3 { + return false + } + for _, part := range parts { + if !isDecimalString(part) { + return false + } + } + return true +} + func isGHPRDiff(args []string) bool { return len(args) >= 2 && args[0] == "pr" && args[1] == "diff" } @@ -305,6 +377,14 @@ func normalizeGHAPIRoute(args []string) string { if part == "" { continue } + if index >= 4 && len(parts) > 3 && parts[3] == "contents" { + parts = append(parts[:4], ":path") + break + } + if index >= 5 && len(parts) > 4 && parts[3] == "git" && parts[4] == "ref" { + parts = append(parts[:5], ":ref") + break + } switch { case isDecimalString(part): parts[index] = ":id" @@ -356,6 +436,18 @@ func isDecimalString(value string) bool { return true } +func isHexString(value string) bool { + if value == "" { + return false + } + for _, r := range value { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') && (r < 'A' || r > 'F') { + return false + } + } + return true +} + func mutatingGHCommand(args []string) bool { if len(args) < 2 { return false diff --git a/internal/cli/gh_shim_cache_test.go b/internal/cli/gh_shim_cache_test.go index b2ecc6a..d79ad2f 100644 --- a/internal/cli/gh_shim_cache_test.go +++ b/internal/cli/gh_shim_cache_test.go @@ -146,11 +146,11 @@ func TestGHShimCommandAwareCacheTTLs(t *testing.T) { if got := ghCommandCacheTTL([]string{"run", "view", "123", "--log"}); got != 12*time.Hour { t.Fatalf("run log ttl = %s, want 12h", got) } - if got := ghCommandCacheTTL([]string{"run", "view", "123", "--job", "456"}); got != 5*time.Minute { - t.Fatalf("run job ttl = %s, want 5m", got) + if got := ghCommandCacheTTL([]string{"run", "view", "123", "--job", "456"}); got != time.Minute { + t.Fatalf("run job ttl = %s, want 1m", got) } - if got := ghCommandCacheTTL([]string{"run", "list", "-R", "openclaw/openclaw"}); got != 2*time.Minute { - t.Fatalf("run list ttl = %s, want 2m", got) + if got := ghCommandCacheTTL([]string{"run", "list", "-R", "openclaw/openclaw"}); got != 30*time.Second { + t.Fatalf("run list ttl = %s, want 30s", got) } if got := ghCommandCacheTTL([]string{"search", "issues", "cache"}); got != 15*time.Minute { t.Fatalf("search ttl = %s, want 15m", got) @@ -158,8 +158,26 @@ func TestGHShimCommandAwareCacheTTLs(t *testing.T) { if got := ghCommandCacheTTL([]string{"api", "-i", "repos/openclaw/openclaw/actions/runs/123/logs"}); got != 12*time.Hour { t.Fatalf("actions log api ttl = %s, want 12h", got) } - if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/actions/runs/123"}); got != 2*time.Minute { - t.Fatalf("actions run api ttl = %s, want 2m", got) + if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/actions/runs/123"}); got != 30*time.Second { + t.Fatalf("actions run api ttl = %s, want 30s", got) + } + if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/pages"}); got != 30*time.Minute { + t.Fatalf("pages api ttl = %s, want 30m", got) + } + if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/contents/README.md?ref=v0.2.0"}); got != 7*24*time.Hour { + t.Fatalf("tagged contents api ttl = %s, want 7d", got) + } + if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/contents/README.md?ref=refs%2Ftags%2Fv0.2.0"}); got != 7*24*time.Hour { + t.Fatalf("refs/tags contents api ttl = %s, want 7d", got) + } + if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/contents/README.md?ref=0123456789abcdef0123456789abcdef01234567"}); got != 7*24*time.Hour { + t.Fatalf("sha contents api ttl = %s, want 7d", got) + } + if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/contents/README.md?ref=vnext"}); got != 30*time.Minute { + t.Fatalf("mutable vnext contents api ttl = %s, want 30m", got) + } + if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/contents/README.md?ref=refs%2Fheads%2Fv0.2.0"}); got != 30*time.Minute { + t.Fatalf("v-prefixed branch contents api ttl = %s, want 30m", got) } if got := normalizeGHAPIRoute([]string{"repos/openclaw/openclaw/actions/runs?per_page=1"}); got != "api repos/:owner/:repo/actions/runs" { t.Fatalf("normalized actions route = %q", got) @@ -167,6 +185,9 @@ func TestGHShimCommandAwareCacheTTLs(t *testing.T) { if got := normalizeGHAPIRoute([]string{"--paginate", "repos/openclaw/openclaw/issues?state=all&creator=octocat", "--jq", ".[].number"}); got != "api repos/:owner/:repo/issues" { t.Fatalf("normalized paginated issues route = %q", got) } + if got := normalizeGHAPIRoute([]string{"repos/openclaw/openclaw/contents/.github/workflows/ci.yml?ref=main"}); got != "api repos/:owner/:repo/contents/:path" { + t.Fatalf("normalized contents route = %q", got) + } entry := ghCommandCacheEntry{CreatedAt: time.Now().Add(-3 * time.Minute), ExitCode: 1, Stderr: "HTTP 403: API rate limit exceeded"} if ttl := ghCommandCacheEntryTTL(entry, 12*time.Hour); ttl != 2*time.Minute { t.Fatalf("rate-limit error ttl = %s, want 2m", ttl) @@ -187,6 +208,14 @@ func TestGHShimCommandAwareCacheTTLs(t *testing.T) { if ttl := ghCommandCacheEntryTTL(completedRuns, 2*time.Minute); ttl != 30*time.Minute { t.Fatalf("completed run list ttl = %s, want 30m", ttl) } + completedJobs := ghCommandCacheEntry{ + Args: []string{"api", "repos/openclaw/openclaw/actions/runs/123/jobs"}, + ExitCode: 0, + Stdout: `{"jobs":[{"status":"completed","conclusion":"success"}]}`, + } + if ttl := ghCommandCacheEntryTTL(completedJobs, time.Minute); ttl != 12*time.Hour { + t.Fatalf("completed jobs ttl = %s, want 12h", ttl) + } } func TestGHShimCanonicalizesEquivalentCacheKeys(t *testing.T) { @@ -350,6 +379,66 @@ func TestGHShimTracksBackendMissesByCommandAndRoute(t *testing.T) { if stats.Counters.BackendMissesByRoute["api repos/:owner/:repo/actions/runs/:id/logs"] != 1 { t.Fatalf("backend misses by route = %#v", stats.Counters.BackendMissesByRoute) } + if stats.Counters.BackendMissesByKey["api repos/openclaw/openclaw/actions/runs/123/logs -i"] != 1 { + t.Fatalf("backend misses by key = %#v", stats.Counters.BackendMissesByKey) + } +} + +func TestGHShimXCacheStatsSinceAndSnapshot(t *testing.T) { + ctx := context.Background() + configPath := seedGHShimRepo(t, ctx) + dir := t.TempDir() + ghPath := filepath.Join(dir, "gh") + if err := os.WriteFile(ghPath, []byte("#!/bin/sh\necho repo:$*\n"), 0o755); err != nil { + t.Fatalf("write fake gh: %v", err) + } + t.Setenv("GITCRAWL_GH_PATH", ghPath) + t.Setenv("GH_REPO", "stats-since/"+filepath.Base(dir)) + t.Setenv("GITCRAWL_GH_CACHE_TTL", "1m") + + run := New() + var stdout bytes.Buffer + run.Stdout = &stdout + args := []string{"--config", configPath, "gh", "repo", "view", "openclaw/gitcrawl", "--json", "nameWithOwner"} + if err := run.Run(ctx, args); err != nil { + t.Fatalf("repo view: %v", err) + } + stdout.Reset() + if err := run.Run(ctx, []string{"--config", configPath, "gh", "xcache", "stats", "--since", "1h", "--json"}); err != nil { + t.Fatalf("xcache stats --since: %v", err) + } + var stats ghCommandCacheStats + if err := json.Unmarshal(stdout.Bytes(), &stats); err != nil { + t.Fatalf("decode stats: %v\n%s", err, stdout.String()) + } + if stats.Since != "1h0m0s" || stats.CumulativeCounters == nil || stats.Counters.BackendMisses != 1 { + t.Fatalf("since stats = %+v", stats) + } + + stdout.Reset() + if err := run.Run(ctx, []string{"--config", configPath, "gh", "xcache", "snapshot", "--reset", "--json"}); err != nil { + t.Fatalf("xcache snapshot: %v", err) + } + var snap ghCommandCacheSnapshotResult + if err := json.Unmarshal(stdout.Bytes(), &snap); err != nil { + t.Fatalf("decode snapshot: %v\n%s", err, stdout.String()) + } + if snap.SnapshotPath == "" || !snap.Reset { + t.Fatalf("snapshot result = %+v", snap) + } + if _, err := os.Stat(snap.SnapshotPath); err != nil { + t.Fatalf("snapshot file: %v", err) + } + stdout.Reset() + if err := run.Run(ctx, []string{"--config", configPath, "gh", "xcache", "stats", "--json"}); err != nil { + t.Fatalf("xcache stats after snapshot reset: %v", err) + } + if err := json.Unmarshal(stdout.Bytes(), &stats); err != nil { + t.Fatalf("decode reset stats: %v\n%s", err, stdout.String()) + } + if stats.Counters.BackendMisses != 0 { + t.Fatalf("snapshot reset counters = %+v", stats.Counters) + } } func TestGHShimCachesReadOnlyFallbackErrors(t *testing.T) { @@ -466,6 +555,80 @@ exit 1 } } +func TestGHShimServesStaleWhileAnotherProcessRefreshes(t *testing.T) { + ctx := context.Background() + configPath := seedGHShimRepo(t, ctx) + dir := t.TempDir() + countPath := filepath.Join(dir, "count") + ghPath := filepath.Join(dir, "gh") + script := `#!/bin/sh +count=0 +if [ -f "$GH_SHIM_COUNT" ]; then + count=$(cat "$GH_SHIM_COUNT") +fi +count=$((count + 1)) +printf "%s" "$count" > "$GH_SHIM_COUNT" +if [ "$count" != "1" ]; then + sleep 1 +fi +echo "release-$count" +` + if err := os.WriteFile(ghPath, []byte(script), 0o755); err != nil { + t.Fatalf("write fake gh: %v", err) + } + t.Setenv("GITCRAWL_GH_PATH", ghPath) + t.Setenv("GH_SHIM_COUNT", countPath) + t.Setenv("GITCRAWL_GH_CACHE_TTL", "1ns") + t.Setenv("GITCRAWL_GH_STALE_GRACE", "1h") + + args := []string{"--config", configPath, "gh", "release", "view", "v1", "-R", "openclaw/openclaw"} + run := New() + var stdout bytes.Buffer + run.Stdout = &stdout + if err := run.Run(ctx, args); err != nil { + t.Fatalf("seed read: %v", err) + } + stdout.Reset() + + var wg sync.WaitGroup + outputs := make(chan string, 2) + errs := make(chan error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + run := New() + var out bytes.Buffer + run.Stdout = &out + if err := run.Run(ctx, args); err != nil { + errs <- err + return + } + outputs <- strings.TrimSpace(out.String()) + }() + } + wg.Wait() + close(errs) + close(outputs) + for err := range errs { + t.Fatalf("stale while refresh run: %v", err) + } + seen := map[string]int{} + for out := range outputs { + seen[out]++ + } + if seen["release-1"] != 1 || seen["release-2"] != 1 { + t.Fatalf("outputs = %#v, want one stale and one refresh", seen) + } + countData, err := os.ReadFile(countPath) + if err != nil { + t.Fatalf("read count: %v", err) + } + if strings.TrimSpace(string(countData)) != "2" { + t.Fatalf("fake gh call count = %q, want 2", countData) + } +} + func TestGHShimMutatingFallbackClearsMatchingCacheForGHXStyleMutations(t *testing.T) { ctx := context.Background() configPath := seedGHShimRepo(t, ctx) diff --git a/internal/cli/gh_shim_descriptor.go b/internal/cli/gh_shim_descriptor.go index f19e78c..2159b31 100644 --- a/internal/cli/gh_shim_descriptor.go +++ b/internal/cli/gh_shim_descriptor.go @@ -347,6 +347,12 @@ func ghCompletedRunCacheTTL(entry ghCommandCacheEntry) time.Duration { } if entry.Args[0] == "api" { route := normalizeGHAPIRoute(entry.Args[1:]) + if strings.Contains(route, "/actions/runs/:id/jobs") && ghJSONJobsCompleted(entry.Stdout) { + return 12 * time.Hour + } + if strings.Contains(route, "/actions/jobs/:id") && ghJSONStatusCompleted(entry.Stdout) { + return 12 * time.Hour + } if strings.Contains(route, "/actions/runs/:id") && ghJSONStatusCompleted(entry.Stdout) { return 12 * time.Hour } @@ -357,6 +363,16 @@ func ghCompletedRunCacheTTL(entry ghCommandCacheEntry) time.Duration { return 0 } +func ghJSONJobsCompleted(raw string) bool { + var payload struct { + Jobs []map[string]any `json:"jobs"` + } + if err := json.Unmarshal([]byte(raw), &payload); err == nil { + return len(payload.Jobs) > 0 && allGHStatusMapsCompleted(payload.Jobs) + } + return ghJSONCollectionCompleted(raw) +} + func ghJSONStatusCompleted(raw string) bool { var payload map[string]any if err := json.Unmarshal([]byte(raw), &payload); err != nil { diff --git a/internal/cli/gh_shim_metrics.go b/internal/cli/gh_shim_metrics.go index a4b6743..e75ddfd 100644 --- a/internal/cli/gh_shim_metrics.go +++ b/internal/cli/gh_shim_metrics.go @@ -4,17 +4,32 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "time" ) type ghXCacheCounters struct { - LocalHits int64 `json:"local_hits"` - FallbackHits int64 `json:"fallback_hits"` - StaleHits int64 `json:"stale_hits"` - BackendMisses int64 `json:"backend_misses"` - PassThroughWrites int64 `json:"pass_through_writes"` + LocalHits int64 `json:"local_hits"` + FallbackHits int64 `json:"fallback_hits"` + StaleHits int64 `json:"stale_hits"` + BackendMisses int64 `json:"backend_misses"` + PassThroughWrites int64 `json:"pass_through_writes"` + BackendMissesByCommand map[string]int64 `json:"backend_misses_by_command,omitempty"` + BackendMissesByRoute map[string]int64 `json:"backend_misses_by_route,omitempty"` + BackendMissesByKey map[string]int64 `json:"backend_misses_by_key,omitempty"` + Hourly map[string]ghXCacheCounterBucket `json:"hourly,omitempty"` +} + +type ghXCacheCounterBucket struct { + StartedAt time.Time `json:"started_at"` + LocalHits int64 `json:"local_hits,omitempty"` + FallbackHits int64 `json:"fallback_hits,omitempty"` + StaleHits int64 `json:"stale_hits,omitempty"` + BackendMisses int64 `json:"backend_misses,omitempty"` + PassThroughWrites int64 `json:"pass_through_writes,omitempty"` BackendMissesByCommand map[string]int64 `json:"backend_misses_by_command,omitempty"` BackendMissesByRoute map[string]int64 `json:"backend_misses_by_route,omitempty"` + BackendMissesByKey map[string]int64 `json:"backend_misses_by_key,omitempty"` } func (a *App) ghXCacheCounters() (ghXCacheCounters, error) { @@ -57,6 +72,28 @@ func (a *App) incrementGHXCacheCounterWithArgs(name string, args []string) error _ = os.Remove(lockPath) }() stats := readGHXCacheCounters(path) + if !incrementGHXCacheCounters(&stats, name, args) { + return nil + } + bucketKey, bucketStart := ghXCacheCurrentBucket(time.Now()) + if stats.Hourly == nil { + stats.Hourly = map[string]ghXCacheCounterBucket{} + } + bucket := stats.Hourly[bucketKey] + if bucket.StartedAt.IsZero() { + bucket.StartedAt = bucketStart + } + _ = incrementGHXCacheCounterBucket(&bucket, name, args) + stats.Hourly[bucketKey] = bucket + pruneGHXCacheBuckets(stats.Hourly, time.Now().Add(-7*24*time.Hour)) + data, err := json.Marshal(stats) + if err != nil { + return err + } + return writeAtomicFile(path, data, 0o600) +} + +func incrementGHXCacheCounters(stats *ghXCacheCounters, name string, args []string) bool { switch name { case "local_hits": stats.LocalHits++ @@ -66,29 +103,69 @@ func (a *App) incrementGHXCacheCounterWithArgs(name string, args []string) error stats.StaleHits++ case "backend_misses": stats.BackendMisses++ - if len(args) > 0 { - if stats.BackendMissesByCommand == nil { - stats.BackendMissesByCommand = map[string]int64{} - } - command := ghCommandName(args) - stats.BackendMissesByCommand[command]++ - if route := ghCommandRoute(args); route != "" { - if stats.BackendMissesByRoute == nil { - stats.BackendMissesByRoute = map[string]int64{} - } - stats.BackendMissesByRoute[route]++ - } - } + incrementGHXCacheMissMaps(&stats.BackendMissesByCommand, &stats.BackendMissesByRoute, &stats.BackendMissesByKey, args) case "pass_through_writes": stats.PassThroughWrites++ default: - return nil + return false } - data, err := json.Marshal(stats) - if err != nil { - return err + return true +} + +func incrementGHXCacheCounterBucket(bucket *ghXCacheCounterBucket, name string, args []string) bool { + switch name { + case "local_hits": + bucket.LocalHits++ + case "fallback_hits": + bucket.FallbackHits++ + case "stale_hits": + bucket.StaleHits++ + case "backend_misses": + bucket.BackendMisses++ + incrementGHXCacheMissMaps(&bucket.BackendMissesByCommand, &bucket.BackendMissesByRoute, &bucket.BackendMissesByKey, args) + case "pass_through_writes": + bucket.PassThroughWrites++ + default: + return false } - return writeAtomicFile(path, data, 0o600) + return true +} + +func incrementGHXCacheMissMaps(byCommand, byRoute, byKey *map[string]int64, args []string) { + if len(args) == 0 { + return + } + if *byCommand == nil { + *byCommand = map[string]int64{} + } + (*byCommand)[ghCommandName(args)]++ + if route := ghCommandRoute(args); route != "" { + if *byRoute == nil { + *byRoute = map[string]int64{} + } + (*byRoute)[route]++ + } + if key := ghCommandMissKey(args); key != "" { + if *byKey == nil { + *byKey = map[string]int64{} + } + (*byKey)[key]++ + } +} + +func ghCommandMissKey(args []string) string { + if len(args) == 0 { + return "" + } + canonical := canonicalGHCommandArgs(args) + if len(canonical) == 0 { + return ghCommandName(args) + } + key := strings.Join(canonical, " ") + if len(key) > 180 { + key = key[:177] + "..." + } + return key } func ghCommandRoute(args []string) string { @@ -116,6 +193,53 @@ func readGHXCacheCounters(path string) ghXCacheCounters { return stats } +func (c ghXCacheCounters) since(since time.Duration, now time.Time) ghXCacheCounters { + if since <= 0 { + return c + } + cutoff := now.Add(-since) + var out ghXCacheCounters + for _, bucket := range c.Hourly { + if bucket.StartedAt.IsZero() || bucket.StartedAt.Before(cutoff) { + continue + } + out.LocalHits += bucket.LocalHits + out.FallbackHits += bucket.FallbackHits + out.StaleHits += bucket.StaleHits + out.BackendMisses += bucket.BackendMisses + out.PassThroughWrites += bucket.PassThroughWrites + mergeCounterMap(&out.BackendMissesByCommand, bucket.BackendMissesByCommand) + mergeCounterMap(&out.BackendMissesByRoute, bucket.BackendMissesByRoute) + mergeCounterMap(&out.BackendMissesByKey, bucket.BackendMissesByKey) + } + return out +} + +func mergeCounterMap(dst *map[string]int64, src map[string]int64) { + if len(src) == 0 { + return + } + if *dst == nil { + *dst = map[string]int64{} + } + for key, value := range src { + (*dst)[key] += value + } +} + +func ghXCacheCurrentBucket(now time.Time) (string, time.Time) { + start := now.UTC().Truncate(time.Hour) + return start.Format("2006-01-02T15:00:00Z"), start +} + +func pruneGHXCacheBuckets(buckets map[string]ghXCacheCounterBucket, cutoff time.Time) { + for key, bucket := range buckets { + if !bucket.StartedAt.IsZero() && bucket.StartedAt.Before(cutoff) { + delete(buckets, key) + } + } +} + func writeAtomicFile(path string, data []byte, perm os.FileMode) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err diff --git a/internal/cli/gh_shim_xcache.go b/internal/cli/gh_shim_xcache.go index 4243336..043a76b 100644 --- a/internal/cli/gh_shim_xcache.go +++ b/internal/cli/gh_shim_xcache.go @@ -13,16 +13,18 @@ import ( ) type ghCommandCacheStats struct { - CacheDir string `json:"cache_dir"` - Entries int `json:"entries"` - Expired int `json:"expired"` - Locks int `json:"locks"` - Bytes int64 `json:"bytes"` - CacheHits int64 `json:"cache_hits"` - TotalReads int64 `json:"total_reads"` - HitRatePercent float64 `json:"hit_rate_percent"` - Counters ghXCacheCounters `json:"counters"` - Commands map[string]ghCommandCacheCount `json:"commands"` + CacheDir string `json:"cache_dir"` + Entries int `json:"entries"` + Expired int `json:"expired"` + Locks int `json:"locks"` + Bytes int64 `json:"bytes"` + Since string `json:"since,omitempty"` + CacheHits int64 `json:"cache_hits"` + TotalReads int64 `json:"total_reads"` + HitRatePercent float64 `json:"hit_rate_percent"` + Counters ghXCacheCounters `json:"counters"` + CumulativeCounters *ghXCacheCounters `json:"cumulative_counters,omitempty"` + Commands map[string]ghCommandCacheCount `json:"commands"` } type ghCommandCacheCount struct { @@ -43,18 +45,28 @@ type ghCommandCacheKeyInfo struct { func (a *App) runGHXCache(args []string) error { if len(args) == 0 { - return usageErr(fmt.Errorf("usage: gh xcache ")) + return usageErr(fmt.Errorf("usage: gh xcache ")) } fs := flag.NewFlagSet("xcache "+args[0], flag.ContinueOnError) fs.SetOutput(io.Discard) jsonOut := fs.Bool("json", false, "write JSON output") + sinceRaw := fs.String("since", "", "show stats for the recent duration (stats only)") + resetAfterSnapshot := fs.Bool("reset", false, "reset counters after writing a snapshot (snapshot only)") if err := fs.Parse(args[1:]); err != nil { return usageErr(err) } a.applyCommandJSON(*jsonOut) switch args[0] { case "stats": - return a.runGHXCacheStats() + var since time.Duration + if strings.TrimSpace(*sinceRaw) != "" { + parsed, err := time.ParseDuration(strings.TrimSpace(*sinceRaw)) + if err != nil || parsed <= 0 { + return usageErr(fmt.Errorf("invalid --since duration %q", *sinceRaw)) + } + since = parsed + } + return a.runGHXCacheStats(since) case "keys": return a.runGHXCacheKeys() case "gc": @@ -63,13 +75,15 @@ func (a *App) runGHXCache(args []string) error { return a.runGHXCacheFlush() case "reset": return a.runGHXCacheReset() + case "snapshot": + return a.runGHXCacheSnapshot(*resetAfterSnapshot) default: return usageErr(fmt.Errorf("unknown xcache command %q", args[0])) } } -func (a *App) runGHXCacheStats() error { - stats, err := a.ghCommandCacheStats() +func (a *App) runGHXCacheStats(since time.Duration) error { + stats, err := a.ghCommandCacheStats(since) if err != nil { return err } @@ -87,11 +101,15 @@ func (a *App) runGHXCacheStats() error { _, _ = fmt.Fprintf(a.Stdout, " %-16s %d entries / %d bytes\n", command, count.Entries, count.Bytes) } } + if stats.Since != "" { + _, _ = fmt.Fprintf(a.Stdout, "\nSince: %s\n", stats.Since) + } _, _ = fmt.Fprintf(a.Stdout, "\nCounters:\n local hits: %d\n fallback hits: %d\n stale hits: %d\n backend misses: %d\n pass-through writes: %d\n hit rate: %.1f%% (%d/%d reads)\n", stats.Counters.LocalHits, stats.Counters.FallbackHits, stats.Counters.StaleHits, stats.Counters.BackendMisses, stats.Counters.PassThroughWrites, stats.HitRatePercent, stats.CacheHits, stats.TotalReads) printGHXCacheMisses(a.Stdout, "Backend Misses by Command", stats.Counters.BackendMissesByCommand) printGHXCacheMisses(a.Stdout, "Backend Misses by Route", stats.Counters.BackendMissesByRoute) + printGHXCacheMisses(a.Stdout, "Backend Misses by Key", stats.Counters.BackendMissesByKey) return nil } @@ -161,6 +179,48 @@ func (a *App) runGHXCacheReset() error { return err } +type ghCommandCacheSnapshotResult struct { + SnapshotPath string `json:"snapshot_path"` + Reset bool `json:"reset"` +} + +func (a *App) runGHXCacheSnapshot(reset bool) error { + stats, err := a.ghCommandCacheStats(0) + if err != nil { + return err + } + dir, err := a.ghCommandCacheDir() + if err != nil { + return err + } + snapshotDir := filepath.Join(dir, "_snapshots") + if err := os.MkdirAll(snapshotDir, 0o755); err != nil { + return err + } + path := filepath.Join(snapshotDir, time.Now().UTC().Format("20060102T150405Z")+".json") + data, err := json.MarshalIndent(stats, "", " ") + if err != nil { + return err + } + if err := writeAtomicFile(path, data, 0o600); err != nil { + return err + } + if reset { + if err := a.resetGHXCacheCounters(); err != nil { + return err + } + } + result := ghCommandCacheSnapshotResult{SnapshotPath: path, Reset: reset} + if a.format == FormatJSON { + return a.writeJSONValue(result, "") + } + _, err = fmt.Fprintf(a.Stdout, "Wrote xcache snapshot: %s\n", path) + if err == nil && reset { + _, err = fmt.Fprintln(a.Stdout, "Reset xcache counters") + } + return err +} + type ghCommandCacheGCResult struct { Removed int `json:"removed"` LocksRemoved int `json:"locks_removed"` @@ -178,7 +238,7 @@ func (a *App) runGHXCacheGC() error { return err } -func (a *App) ghCommandCacheStats() (ghCommandCacheStats, error) { +func (a *App) ghCommandCacheStats(since time.Duration) (ghCommandCacheStats, error) { dir, err := a.ghCommandCacheDir() if err != nil { return ghCommandCacheStats{}, err @@ -188,9 +248,15 @@ func (a *App) ghCommandCacheStats() (ghCommandCacheStats, error) { return ghCommandCacheStats{}, err } counters, _ := a.ghXCacheCounters() + cumulative := counters stats := ghCommandCacheStats{CacheDir: dir, Locks: locks, Counters: counters, Commands: map[string]ghCommandCacheCount{}} - stats.CacheHits = counters.LocalHits + counters.FallbackHits + counters.StaleHits - stats.TotalReads = stats.CacheHits + counters.BackendMisses + if since > 0 { + stats.Since = since.String() + stats.CumulativeCounters = &cumulative + stats.Counters = counters.since(since, time.Now()) + } + stats.CacheHits = stats.Counters.LocalHits + stats.Counters.FallbackHits + stats.Counters.StaleHits + stats.TotalReads = stats.CacheHits + stats.Counters.BackendMisses if stats.TotalReads > 0 { stats.HitRatePercent = float64(stats.CacheHits) / float64(stats.TotalReads) * 100 }