feat: improve gh shim cache observability
This commit is contained in:
Peter Steinberger
parent
5e441a9e48
commit
e5621d1b78
@ -2,6 +2,8 @@
|
||||
|
||||
## 0.3.0 - Unreleased
|
||||
|
||||
- Improve `gh` shim cache coordination and observability with stale-while-revalidate reads, finer Actions/API TTLs, recent-window stats, top miss keys, and `xcache snapshot`.
|
||||
|
||||
## 0.2.0 - 2026-05-05
|
||||
|
||||
- Add Homebrew tap installation via `brew install openclaw/tap/gitcrawl`.
|
||||
|
||||
@ -52,7 +52,7 @@ gitcrawl tui owner/repo
|
||||
Pass `--numbers` to refresh exact issue or pull request rows without relying on list ordering or updated-time windows.
|
||||
Pass `--with pr-details` or `--include-pr-details` to hydrate pull request files, commits, checks, and workflow runs for local review. The `gh` shim can also auto-hydrate one exact PR on a PR-detail miss, then retry locally.
|
||||
`gitcrawl search issues|prs` accepts the common `gh search` shape (`<query> -R owner/repo --state open --json fields --limit N`) and answers from the local SQLite cache. It is intended for discovery without spending GitHub REST search quota; use `gh` for final live verification and GitHub write actions. Pass `--sync-if-stale 5m` to perform one metadata sync before the cached search when the local repository mirror is older than that duration.
|
||||
`gitcrawl gh` is a gh-compatible shim for agent workflows. It answers broad `gh search issues|prs`, `gh issue/pr list`, supported `gh issue/pr view --json` fields, hydrated `gh pr checks`, and hydrated `gh run list/view` from local SQLite, then falls through to the real GitHub CLI for unsupported commands. Local `gh issue/pr list` supports common filters such as `--author`, `--assignee`, and repeated `--label`. Read-only fallthroughs such as `gh pr diff`, `gh repo view/list`, `gh release list/view`, `gh workflow list/view`, `gh secret list`, `gh variable get/list`, `gh label list`, read-only `gh search` kinds, GET-only REST `gh api` calls, and read-only `gh api graphql` queries use a command-aware persistent cache under `cache/gh-shim`; Actions run/job logs get longer TTLs, completed run views are kept much longer than active CI status, user profile reads get a 7-day TTL, read-only GraphQL gets a 6-hour TTL, and `gh pr diff` entries are keyed by the cached PR head SHA when available. Explicit API paths and explicit repositories share cache entries across sibling checkouts even when agents set different `GH_REPO` values; implicit repo reads stay isolated by `GH_REPO` or current working directory. Cache keys canonicalize common flags such as `-R`/`--repo` and sorted `--json` fields so equivalent agent commands coalesce. Repeat read failures are cached by default so agents do not rediscover the same missing release or workflow, but rate-limit error entries expire quickly; if GitHub rate-limits a refresh and an expired successful entry exists, the shim serves the stale response with a warning instead of failing the read. Set `GITCRAWL_GH_CACHE_ERRORS=0` to disable error caching. Mutating commands pass through, increment write counters, and invalidate matching cache tags instead of flushing unrelated entries. `gh xcache stats|keys|gc|flush|reset` inspects, garbage-collects, clears, or resets fallthrough-cache counters, including hit rate plus per-command and per-route backend miss counters. Set `GITCRAWL_GH_PATH` to choose the backend `gh`, and symlink or install the binary as `gh`/`gitcrawl-gh` to run the shim directly.
|
||||
`gitcrawl gh` is a gh-compatible shim for agent workflows. It answers broad `gh search issues|prs`, `gh issue/pr list`, supported `gh issue/pr view --json` fields, hydrated `gh pr checks`, and hydrated `gh run list/view` from local SQLite, then falls through to the real GitHub CLI for unsupported commands. Local `gh issue/pr list` supports common filters such as `--author`, `--assignee`, and repeated `--label`. Read-only fallthroughs such as `gh pr diff`, `gh repo view/list`, `gh release list/view`, `gh workflow list/view`, `gh secret list`, `gh variable get/list`, `gh label list`, read-only `gh search` kinds, GET-only REST `gh api` calls, and read-only `gh api graphql` queries use a command-aware persistent cache under `cache/gh-shim`; Actions run/job logs get longer TTLs, completed run/job reads are kept much longer than active CI status, user profile reads get a 7-day TTL, read-only GraphQL gets a 6-hour TTL, and `gh pr diff` entries are keyed by the cached PR head SHA when available. Explicit API paths and explicit repositories share cache entries across sibling checkouts even when agents set different `GH_REPO` values; implicit repo reads stay isolated by `GH_REPO` or current working directory. Cache keys canonicalize common flags such as `-R`/`--repo` and sorted `--json` fields so equivalent agent commands coalesce. Repeat read failures are cached by default so agents do not rediscover the same missing release or workflow, but rate-limit error entries expire quickly; if GitHub rate-limits a refresh and an expired successful entry exists, the shim serves the stale response with a warning instead of failing the read. When another process is refreshing an expired successful entry, peers may serve stale inside a short grace window instead of joining the backend stampede. Set `GITCRAWL_GH_STALE_GRACE=0` to disable stale-while-revalidate, or `GITCRAWL_GH_CACHE_ERRORS=0` to disable error caching. Mutating commands pass through, increment write counters, and invalidate matching cache tags instead of flushing unrelated entries. `gh xcache stats|keys|gc|flush|reset|snapshot` inspects, garbage-collects, clears, resets, or snapshots fallthrough-cache counters, including hit rate plus per-command, per-route, per-key, and `--since` recent-window miss counters. Set `GITCRAWL_GH_PATH` to choose the backend `gh`, and symlink or install the binary as `gh`/`gitcrawl-gh` to run the shim directly.
|
||||
The TUI starts at `--min-size 5` and `--sort size`, like ghcrawl's saved default, so the first screen is the useful cluster workload instead of singleton noise. Pass `--min-size 1` when you intentionally want singleton clusters. Mouse support is built in: click rows, wheel panes, and right-click for copy, sort, filter, jump, link, neighbor, local close/reopen, and member triage actions. Press `a` to open the same action menu from the keyboard, `#` to jump directly to an issue or PR number, `p` to switch between repositories already present in the local store, or `n` to load neighbors for the selected issue or PR. Enter from the members pane also loads neighbors before opening detail. The TUI quietly refreshes from the local store every 15 seconds.
|
||||
|
||||
## Local Defaults
|
||||
|
||||
3
SPEC.md
3
SPEC.md
@ -122,9 +122,10 @@ gitcrawl gh xcache stats
|
||||
gitcrawl gh xcache keys
|
||||
gitcrawl gh xcache reset
|
||||
gitcrawl gh xcache flush
|
||||
gitcrawl gh xcache snapshot [--reset]
|
||||
```
|
||||
|
||||
The cache key includes the resolved gitcrawl config path, current working directory, `GH_HOST`, `GH_REPO`, stable PR-diff identity when available, and canonicalized `gh` arguments. This keeps sibling checkouts and portable stores isolated while still coalescing equivalent agent calls such as reordered flags or sorted `--json` fields. Concurrent cache misses use a lock file so one process populates the entry while peers wait for the result.
|
||||
The cache key includes the resolved gitcrawl config path, current working directory, `GH_HOST`, `GH_REPO`, stable PR-diff identity when available, and canonicalized `gh` arguments. This keeps sibling checkouts and portable stores isolated while still coalescing equivalent agent calls such as reordered flags or sorted `--json` fields. Concurrent cache misses use a lock file so one process populates the entry while peers wait for the result; if an expired successful entry is still inside its stale grace window, peers may serve stale while the lock holder refreshes it. `xcache stats --since <duration>` reports recent-window counters from hourly buckets, and miss maps include command, normalized route, and canonical key views.
|
||||
|
||||
## Config
|
||||
|
||||
|
||||
@ -122,6 +122,10 @@ gh issue comment 456 -R owner/repo --body "Duplicate of #123"
|
||||
# Periodically log cache stats — watch local_hits climb relative to backend_misses.
|
||||
gitcrawl gh xcache stats --json \
|
||||
| jq '{local: .counters.local_hits, fallback: .counters.fallback_hits, github: .counters.backend_misses}'
|
||||
|
||||
# During release/debug sessions, compare a recent window or snapshot before reset.
|
||||
gitcrawl gh xcache stats --since 1h --json
|
||||
gitcrawl gh xcache snapshot --reset --json
|
||||
```
|
||||
|
||||
## Multi-repo automation
|
||||
|
||||
@ -101,7 +101,7 @@ These work on every command.
|
||||
| `gitcrawl gh label list ...` | Falls through; cached briefly | [gh shim](/gh-shim/) |
|
||||
| `gitcrawl gh api <GET path>` | Falls through; cached briefly (GET-only REST) | [gh shim](/gh-shim/) |
|
||||
| `gitcrawl gh api graphql -f query=...` | Falls through; read-only queries are cached | [gh shim](/gh-shim/#read-only-fallthroughs-cached) |
|
||||
| `gitcrawl gh xcache stats\|keys\|gc\|flush\|reset [--json]` | Cache inspection / housekeeping | [gh shim](/gh-shim/#cache-inspection-xcache) |
|
||||
| `gitcrawl gh xcache stats [--since <duration>] \| keys \| gc \| flush \| reset \| snapshot [--reset] [--json]` | Cache inspection / housekeeping | [gh shim](/gh-shim/#cache-inspection-xcache) |
|
||||
| _Anything else_ | Falls through to real `gh` | [gh shim](/gh-shim/) |
|
||||
|
||||
The shim binary can be installed standalone by symlinking the `gitcrawl` binary as `gh` or `gitcrawl-gh`.
|
||||
|
||||
@ -102,6 +102,7 @@ checkout_dir = "/Users/me/.config/gitcrawl/portable"
|
||||
| `GITCRAWL_GH_PATH` | Path to the real `gh` binary used for fallthrough |
|
||||
| `GITCRAWL_GH_AUTO_HYDRATE` | Set to `0` to disable PR auto-hydration on cache miss |
|
||||
| `GITCRAWL_GH_CACHE_TTL` | Override fallthrough cache TTL (e.g., `5m`, `1h`) |
|
||||
| `GITCRAWL_GH_STALE_GRACE` | Override stale-while-revalidate grace for expired successful fallthrough entries |
|
||||
| `GITCRAWL_GH_CACHE_ERRORS` | Set to `0` to avoid caching non-zero read-only fallthroughs |
|
||||
|
||||
If `GITCRAWL_GH_PATH` is unset, the shim probes common Homebrew install paths and then your `PATH`. Set it explicitly when you symlink the gitcrawl binary as `gh` (otherwise the shim will recurse into itself).
|
||||
|
||||
@ -101,9 +101,9 @@ These commands always run real `gh` but the response body is cached for the next
|
||||
|
||||
Common Actions REST reads such as run status, job lists, and logs get Actions-aware TTLs.
|
||||
|
||||
Default cache TTLs are command-aware: active `gh run list` and run-status reads use `2m`; completed run views are kept for `12h`; completed run lists are kept for `30m`; workflow, job detail, and Actions job-list reads use `5m`; search reads use `15m`; release metadata uses `30m`; GitHub user profile reads use `7d`; read-only GraphQL queries use `6h`; completed-style run/job log reads use `12h`; `gh pr diff` uses `5m` without a stable SHA and `7d` with one. Most other read-only fallthroughs use `5m` to `10m`. Override with `GITCRAWL_GH_CACHE_TTL=5m` or similar.
|
||||
Default cache TTLs are command-aware: active `gh run list` and run-status reads use `30s`; completed run views, completed Actions job lists, and run/job logs are kept for `12h`; completed run lists are kept for `30m`; workflow reads use `15m`; search reads use `15m`; release metadata uses `1h`; GitHub user profile reads use `7d`; read-only GraphQL queries use `6h`; GitHub Pages metadata uses `15m` to `30m`; tagged/SHA `contents` API reads use `7d`; `gh pr diff` uses `5m` without a stable SHA and `7d` with one. Most other read-only fallthroughs use `5m` to `10m`. Override with `GITCRAWL_GH_CACHE_TTL=5m` or similar.
|
||||
|
||||
Repeat read failures are cached by default too. That avoids a fleet of agents all rediscovering the same missing release, workflow, secret, or unsupported field. Error entries are capped to shorter lifetimes, and rate-limit errors are capped at `2m` so a reset is not masked all day. If GitHub returns a rate-limit error while refreshing an expired successful entry, the shim serves that stale success with a warning instead of failing the read. Set `GITCRAWL_GH_CACHE_ERRORS=0` to cache successful reads only.
|
||||
Repeat read failures are cached by default too. That avoids a fleet of agents all rediscovering the same missing release, workflow, secret, or unsupported field. Error entries are capped to shorter lifetimes, and rate-limit errors are capped at `2m` so a reset is not masked all day. If GitHub returns a rate-limit error while refreshing an expired successful entry, the shim serves that stale success with a warning instead of failing the read. When another process is already refreshing an expired successful entry, peers can serve that stale entry within a short command-aware grace window instead of joining the backend stampede. Set `GITCRAWL_GH_STALE_GRACE=0` to disable stale-while-revalidate, or `GITCRAWL_GH_CACHE_ERRORS=0` to cache successful reads only.
|
||||
|
||||
## Auto-hydration
|
||||
|
||||
@ -124,9 +124,10 @@ gitcrawl gh xcache keys # per-entry detail
|
||||
gitcrawl gh xcache gc # remove expired entries + stale lock files
|
||||
gitcrawl gh xcache flush # clear everything
|
||||
gitcrawl gh xcache reset # reset counters without deleting entries
|
||||
gitcrawl gh xcache snapshot # write a counter snapshot for later comparison
|
||||
```
|
||||
|
||||
All accept `--json` for scripting.
|
||||
All accept `--json` for scripting. `stats` accepts `--since 1h` for recent-window counters. `snapshot` accepts `--reset` to checkpoint counters before a noisy release/debugging session.
|
||||
|
||||
`stats` JSON:
|
||||
|
||||
@ -152,6 +153,9 @@ All accept `--json` for scripting.
|
||||
},
|
||||
"backend_misses_by_route": {
|
||||
"api repos/:owner/:repo/actions/runs/:id/logs": 3
|
||||
},
|
||||
"backend_misses_by_key": {
|
||||
"api repos/openclaw/gitcrawl/actions/runs/123/logs -i": 2
|
||||
}
|
||||
},
|
||||
"commands": {
|
||||
@ -161,7 +165,7 @@ All accept `--json` for scripting.
|
||||
}
|
||||
```
|
||||
|
||||
`local_hits` are answered from SQLite; `fallback_hits` are answered from the fallthrough cache; `stale_hits` are expired successful cache entries served after a backend rate-limit response; `backend_misses` actually hit GitHub. The per-command and per-route miss maps show which shapes still escape the cache, which is usually the fastest way to find the next optimization.
|
||||
`local_hits` are answered from SQLite; `fallback_hits` are answered from the fallthrough cache; `stale_hits` are expired successful cache entries served after a backend rate-limit response or while another process refreshes the key; `backend_misses` actually hit GitHub. The per-command, per-route, and per-key miss maps show which shapes still escape the cache, which is usually the fastest way to find the next optimization.
|
||||
|
||||
## Cache key composition
|
||||
|
||||
|
||||
@ -104,17 +104,21 @@ Override the config root with `--config <path>` or `GITCRAWL_CONFIG`.
|
||||
| Cache class | TTL |
|
||||
| --- | --- |
|
||||
| Most read-only fallthroughs | `5m`-`10m` |
|
||||
| `gh run list` / run status | `2m` |
|
||||
| `gh run list` / run status | `30s` |
|
||||
| `gh run view --log` / `--log-failed` | `12h` |
|
||||
| `gh run view --job` | `5m` |
|
||||
| `gh run view --job` | `1m` |
|
||||
| `gh search ...` | `15m` |
|
||||
| `gh release ...` | `30m` |
|
||||
| `gh api` Actions run status | `2m` |
|
||||
| `gh api` Actions job lists / workflow reads | `5m` |
|
||||
| `gh release ...` | `1h` |
|
||||
| `gh api` Actions run status | `30s` |
|
||||
| `gh api` Actions job lists | `1m` active, `12h` completed |
|
||||
| `gh api` workflow reads | `15m` |
|
||||
| `gh api` Actions run/job logs | `12h` |
|
||||
| `gh api` Pages metadata | `15m`-`30m` |
|
||||
| `gh api` tagged/SHA contents | `7d` |
|
||||
| `gh pr diff` without stable head SHA | `5m` |
|
||||
| `gh pr diff` with stable head SHA | `7d` |
|
||||
| Override | `GITCRAWL_GH_CACHE_TTL` |
|
||||
| Stale-while-revalidate grace | command-aware; override with `GITCRAWL_GH_STALE_GRACE` |
|
||||
| Cache read failures | on by default; error TTL is capped (`2m` for rate-limit errors); disable with `GITCRAWL_GH_CACHE_ERRORS=0` |
|
||||
|
||||
## gh shim cache key composition
|
||||
|
||||
@ -41,8 +41,8 @@ func (a *App) execRealGHMaybeCached(ctx context.Context, args []string) error {
|
||||
lockPath := entryPath + ".lock"
|
||||
lock, locked := tryGHCommandCacheLock(lockPath)
|
||||
if !locked {
|
||||
if entry, ok := waitGHCommandCache(entryPath, lockPath, ttl); ok {
|
||||
_ = a.incrementGHXCacheCounter("fallback_hits")
|
||||
if entry, hit, ok := waitGHCommandCache(entryPath, lockPath, ttl, staleEntry, hasStaleEntry); ok {
|
||||
_ = a.incrementGHXCacheCounter(hit)
|
||||
return a.writeGHCommandCacheEntry(entry)
|
||||
}
|
||||
lock, locked = tryGHCommandCacheLock(lockPath)
|
||||
@ -60,7 +60,7 @@ func (a *App) execRealGHMaybeCached(ctx context.Context, args []string) error {
|
||||
|
||||
stdout, stderr, exitCode, err := a.captureRealGH(ctx, args)
|
||||
_ = a.incrementGHXCacheBackendMiss(args)
|
||||
if err != nil && hasStaleEntry && staleEntry.ExitCode == 0 && ghCommandOutputLooksRateLimited(stdout, stderr) {
|
||||
if err != nil && hasStaleEntry && ghCommandCacheEntryCanServeStale(staleEntry, ttl) && ghCommandOutputLooksRateLimited(stdout, stderr) {
|
||||
_ = a.incrementGHXCacheCounter("stale_hits")
|
||||
_, _ = fmt.Fprintf(a.Stderr, "gitcrawl: GitHub rate limited; serving stale cached gh response from %s ago\n", time.Since(staleEntry.CreatedAt).Round(time.Second))
|
||||
return a.writeGHCommandCacheEntry(staleEntry)
|
||||
@ -216,6 +216,47 @@ func ghCommandCacheEntryTTL(entry ghCommandCacheEntry, ttl time.Duration) time.D
|
||||
return ttl
|
||||
}
|
||||
|
||||
func ghCommandCacheEntryCanServeStale(entry ghCommandCacheEntry, ttl time.Duration) bool {
|
||||
if entry.ExitCode != 0 || entry.CreatedAt.IsZero() {
|
||||
return false
|
||||
}
|
||||
age := time.Since(entry.CreatedAt)
|
||||
if age <= ghCommandCacheEntryTTL(entry, ttl) {
|
||||
return true
|
||||
}
|
||||
return age <= ghCommandCacheEntryTTL(entry, ttl)+ghCommandCacheStaleGrace(entry.Args)
|
||||
}
|
||||
|
||||
func ghCommandCacheStaleGrace(args []string) time.Duration {
|
||||
if raw := strings.TrimSpace(os.Getenv("GITCRAWL_GH_STALE_GRACE")); raw != "" {
|
||||
if duration, err := time.ParseDuration(raw); err == nil && duration >= 0 {
|
||||
return duration
|
||||
}
|
||||
}
|
||||
if len(args) == 0 {
|
||||
return 5 * time.Minute
|
||||
}
|
||||
switch args[0] {
|
||||
case "run":
|
||||
return 2 * time.Minute
|
||||
case "api":
|
||||
route := normalizeGHAPIRoute(args[1:])
|
||||
switch {
|
||||
case strings.Contains(route, "/actions/runs"):
|
||||
return 2 * time.Minute
|
||||
case strings.Contains(route, "/pages"):
|
||||
return 30 * time.Minute
|
||||
case strings.Contains(route, "/contents"):
|
||||
return 6 * time.Hour
|
||||
case strings.HasPrefix(route, "api users/"):
|
||||
return 24 * time.Hour
|
||||
}
|
||||
case "release", "workflow", "repo":
|
||||
return 30 * time.Minute
|
||||
}
|
||||
return 10 * time.Minute
|
||||
}
|
||||
|
||||
func ghCommandCacheEntryLooksRateLimited(entry ghCommandCacheEntry) bool {
|
||||
return ghCommandOutputLooksRateLimited(entry.Stdout, entry.Stderr)
|
||||
}
|
||||
@ -270,19 +311,28 @@ func tryGHCommandCacheLock(path string) (*os.File, bool) {
|
||||
return lock, true
|
||||
}
|
||||
|
||||
func waitGHCommandCache(entryPath, lockPath string, ttl time.Duration) (ghCommandCacheEntry, bool) {
|
||||
func waitGHCommandCache(entryPath, lockPath string, ttl time.Duration, staleEntry ghCommandCacheEntry, hasStaleEntry bool) (ghCommandCacheEntry, string, bool) {
|
||||
if hasStaleEntry && ghCommandCacheEntryCanServeStale(staleEntry, ttl) {
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
if entry, ok := readGHCommandCache(entryPath, ttl); ok {
|
||||
return entry, "fallback_hits", true
|
||||
}
|
||||
if _, err := os.Stat(lockPath); err == nil {
|
||||
return staleEntry, "stale_hits", true
|
||||
}
|
||||
}
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if entry, ok := readGHCommandCache(entryPath, ttl); ok {
|
||||
return entry, true
|
||||
return entry, "fallback_hits", true
|
||||
}
|
||||
if _, err := os.Stat(lockPath); os.IsNotExist(err) {
|
||||
return ghCommandCacheEntry{}, false
|
||||
return ghCommandCacheEntry{}, "", false
|
||||
}
|
||||
}
|
||||
_ = os.Remove(lockPath)
|
||||
return ghCommandCacheEntry{}, false
|
||||
return ghCommandCacheEntry{}, "", false
|
||||
}
|
||||
|
||||
func (a *App) ghCommandCacheKey(ctx context.Context, args []string) string {
|
||||
|
||||
@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@ -189,7 +190,7 @@ func ghCommandCacheTTLBase(args []string, stablePRDiff bool) time.Duration {
|
||||
|
||||
func ghRunCacheTTL(args []string) time.Duration {
|
||||
if len(args) == 0 {
|
||||
return 2 * time.Minute
|
||||
return 30 * time.Second
|
||||
}
|
||||
switch args[0] {
|
||||
case "view":
|
||||
@ -197,13 +198,13 @@ func ghRunCacheTTL(args []string) time.Duration {
|
||||
return 12 * time.Hour
|
||||
}
|
||||
if hasAnyGHFlag(args[1:], "--job") {
|
||||
return 5 * time.Minute
|
||||
return 1 * time.Minute
|
||||
}
|
||||
return 2 * time.Minute
|
||||
return 30 * time.Second
|
||||
case "list":
|
||||
return 2 * time.Minute
|
||||
return 30 * time.Second
|
||||
default:
|
||||
return 2 * time.Minute
|
||||
return 30 * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
@ -214,20 +215,35 @@ func ghAPICacheTTL(args []string) time.Duration {
|
||||
return 6 * time.Hour
|
||||
case strings.HasPrefix(route, "api users/"):
|
||||
return 7 * 24 * time.Hour
|
||||
case strings.Contains(route, "/contents"):
|
||||
if ghAPIContentRefIsStable(args) {
|
||||
return 7 * 24 * time.Hour
|
||||
}
|
||||
return 30 * time.Minute
|
||||
case strings.Contains(route, "/pages/builds/latest"):
|
||||
return 2 * time.Minute
|
||||
case strings.Contains(route, "/pages/health"):
|
||||
return 15 * time.Minute
|
||||
case strings.Contains(route, "/pages"):
|
||||
return 30 * time.Minute
|
||||
case strings.Contains(route, "/actions/runs/:id/logs"):
|
||||
return 12 * time.Hour
|
||||
case strings.Contains(route, "/actions/jobs/:id/logs"):
|
||||
return 12 * time.Hour
|
||||
case strings.Contains(route, "/actions/runs/:id/jobs"):
|
||||
return 5 * time.Minute
|
||||
return 1 * time.Minute
|
||||
case strings.Contains(route, "/actions/jobs/:id"):
|
||||
return 1 * time.Minute
|
||||
case strings.Contains(route, "/pending_deployments"):
|
||||
return 30 * time.Second
|
||||
case strings.Contains(route, "/actions/runs/:id"):
|
||||
return 2 * time.Minute
|
||||
return 30 * time.Second
|
||||
case strings.Contains(route, "/actions/workflows/"):
|
||||
return 5 * time.Minute
|
||||
return 15 * time.Minute
|
||||
case strings.Contains(route, "/actions/runs"):
|
||||
return 2 * time.Minute
|
||||
return 30 * time.Second
|
||||
case strings.Contains(route, "/releases"):
|
||||
return 30 * time.Minute
|
||||
return 1 * time.Hour
|
||||
case strings.Contains(route, "/branches") || strings.Contains(route, "/commits"):
|
||||
return 10 * time.Minute
|
||||
default:
|
||||
@ -235,6 +251,62 @@ func ghAPICacheTTL(args []string) time.Duration {
|
||||
}
|
||||
}
|
||||
|
||||
func ghAPIContentRefIsStable(args []string) bool {
|
||||
path := ghAPIPathArg(args)
|
||||
_, rawQuery, found := strings.Cut(path, "?")
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
for _, part := range strings.Split(rawQuery, "&") {
|
||||
name, value, ok := strings.Cut(part, "=")
|
||||
if !ok || name != "ref" {
|
||||
continue
|
||||
}
|
||||
value = strings.TrimSpace(value)
|
||||
if decoded, err := url.QueryUnescape(value); err == nil {
|
||||
value = strings.TrimSpace(decoded)
|
||||
}
|
||||
if len(value) == 40 && isHexString(value) {
|
||||
return true
|
||||
}
|
||||
if ghAPIContentRefIsStableReleaseTag(value) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ghAPIContentRefIsStableReleaseTag(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
if strings.HasPrefix(value, "refs/heads/") {
|
||||
return false
|
||||
}
|
||||
value = strings.TrimPrefix(value, "refs/tags/")
|
||||
if strings.HasPrefix(value, "refs/") {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(value, "v") {
|
||||
value = strings.TrimPrefix(value, "v")
|
||||
}
|
||||
core := value
|
||||
if before, _, found := strings.Cut(core, "+"); found {
|
||||
core = before
|
||||
}
|
||||
if before, _, found := strings.Cut(core, "-"); found {
|
||||
core = before
|
||||
}
|
||||
parts := strings.Split(core, ".")
|
||||
if len(parts) != 3 {
|
||||
return false
|
||||
}
|
||||
for _, part := range parts {
|
||||
if !isDecimalString(part) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isGHPRDiff(args []string) bool {
|
||||
return len(args) >= 2 && args[0] == "pr" && args[1] == "diff"
|
||||
}
|
||||
@ -305,6 +377,14 @@ func normalizeGHAPIRoute(args []string) string {
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
if index >= 4 && len(parts) > 3 && parts[3] == "contents" {
|
||||
parts = append(parts[:4], ":path")
|
||||
break
|
||||
}
|
||||
if index >= 5 && len(parts) > 4 && parts[3] == "git" && parts[4] == "ref" {
|
||||
parts = append(parts[:5], ":ref")
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case isDecimalString(part):
|
||||
parts[index] = ":id"
|
||||
@ -356,6 +436,18 @@ func isDecimalString(value string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func isHexString(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if (r < '0' || r > '9') && (r < 'a' || r > 'f') && (r < 'A' || r > 'F') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func mutatingGHCommand(args []string) bool {
|
||||
if len(args) < 2 {
|
||||
return false
|
||||
|
||||
@ -146,11 +146,11 @@ func TestGHShimCommandAwareCacheTTLs(t *testing.T) {
|
||||
if got := ghCommandCacheTTL([]string{"run", "view", "123", "--log"}); got != 12*time.Hour {
|
||||
t.Fatalf("run log ttl = %s, want 12h", got)
|
||||
}
|
||||
if got := ghCommandCacheTTL([]string{"run", "view", "123", "--job", "456"}); got != 5*time.Minute {
|
||||
t.Fatalf("run job ttl = %s, want 5m", got)
|
||||
if got := ghCommandCacheTTL([]string{"run", "view", "123", "--job", "456"}); got != time.Minute {
|
||||
t.Fatalf("run job ttl = %s, want 1m", got)
|
||||
}
|
||||
if got := ghCommandCacheTTL([]string{"run", "list", "-R", "openclaw/openclaw"}); got != 2*time.Minute {
|
||||
t.Fatalf("run list ttl = %s, want 2m", got)
|
||||
if got := ghCommandCacheTTL([]string{"run", "list", "-R", "openclaw/openclaw"}); got != 30*time.Second {
|
||||
t.Fatalf("run list ttl = %s, want 30s", got)
|
||||
}
|
||||
if got := ghCommandCacheTTL([]string{"search", "issues", "cache"}); got != 15*time.Minute {
|
||||
t.Fatalf("search ttl = %s, want 15m", got)
|
||||
@ -158,8 +158,26 @@ func TestGHShimCommandAwareCacheTTLs(t *testing.T) {
|
||||
if got := ghCommandCacheTTL([]string{"api", "-i", "repos/openclaw/openclaw/actions/runs/123/logs"}); got != 12*time.Hour {
|
||||
t.Fatalf("actions log api ttl = %s, want 12h", got)
|
||||
}
|
||||
if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/actions/runs/123"}); got != 2*time.Minute {
|
||||
t.Fatalf("actions run api ttl = %s, want 2m", got)
|
||||
if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/actions/runs/123"}); got != 30*time.Second {
|
||||
t.Fatalf("actions run api ttl = %s, want 30s", got)
|
||||
}
|
||||
if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/pages"}); got != 30*time.Minute {
|
||||
t.Fatalf("pages api ttl = %s, want 30m", got)
|
||||
}
|
||||
if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/contents/README.md?ref=v0.2.0"}); got != 7*24*time.Hour {
|
||||
t.Fatalf("tagged contents api ttl = %s, want 7d", got)
|
||||
}
|
||||
if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/contents/README.md?ref=refs%2Ftags%2Fv0.2.0"}); got != 7*24*time.Hour {
|
||||
t.Fatalf("refs/tags contents api ttl = %s, want 7d", got)
|
||||
}
|
||||
if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/contents/README.md?ref=0123456789abcdef0123456789abcdef01234567"}); got != 7*24*time.Hour {
|
||||
t.Fatalf("sha contents api ttl = %s, want 7d", got)
|
||||
}
|
||||
if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/contents/README.md?ref=vnext"}); got != 30*time.Minute {
|
||||
t.Fatalf("mutable vnext contents api ttl = %s, want 30m", got)
|
||||
}
|
||||
if got := ghCommandCacheTTL([]string{"api", "repos/openclaw/openclaw/contents/README.md?ref=refs%2Fheads%2Fv0.2.0"}); got != 30*time.Minute {
|
||||
t.Fatalf("v-prefixed branch contents api ttl = %s, want 30m", got)
|
||||
}
|
||||
if got := normalizeGHAPIRoute([]string{"repos/openclaw/openclaw/actions/runs?per_page=1"}); got != "api repos/:owner/:repo/actions/runs" {
|
||||
t.Fatalf("normalized actions route = %q", got)
|
||||
@ -167,6 +185,9 @@ func TestGHShimCommandAwareCacheTTLs(t *testing.T) {
|
||||
if got := normalizeGHAPIRoute([]string{"--paginate", "repos/openclaw/openclaw/issues?state=all&creator=octocat", "--jq", ".[].number"}); got != "api repos/:owner/:repo/issues" {
|
||||
t.Fatalf("normalized paginated issues route = %q", got)
|
||||
}
|
||||
if got := normalizeGHAPIRoute([]string{"repos/openclaw/openclaw/contents/.github/workflows/ci.yml?ref=main"}); got != "api repos/:owner/:repo/contents/:path" {
|
||||
t.Fatalf("normalized contents route = %q", got)
|
||||
}
|
||||
entry := ghCommandCacheEntry{CreatedAt: time.Now().Add(-3 * time.Minute), ExitCode: 1, Stderr: "HTTP 403: API rate limit exceeded"}
|
||||
if ttl := ghCommandCacheEntryTTL(entry, 12*time.Hour); ttl != 2*time.Minute {
|
||||
t.Fatalf("rate-limit error ttl = %s, want 2m", ttl)
|
||||
@ -187,6 +208,14 @@ func TestGHShimCommandAwareCacheTTLs(t *testing.T) {
|
||||
if ttl := ghCommandCacheEntryTTL(completedRuns, 2*time.Minute); ttl != 30*time.Minute {
|
||||
t.Fatalf("completed run list ttl = %s, want 30m", ttl)
|
||||
}
|
||||
completedJobs := ghCommandCacheEntry{
|
||||
Args: []string{"api", "repos/openclaw/openclaw/actions/runs/123/jobs"},
|
||||
ExitCode: 0,
|
||||
Stdout: `{"jobs":[{"status":"completed","conclusion":"success"}]}`,
|
||||
}
|
||||
if ttl := ghCommandCacheEntryTTL(completedJobs, time.Minute); ttl != 12*time.Hour {
|
||||
t.Fatalf("completed jobs ttl = %s, want 12h", ttl)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGHShimCanonicalizesEquivalentCacheKeys(t *testing.T) {
|
||||
@ -350,6 +379,66 @@ func TestGHShimTracksBackendMissesByCommandAndRoute(t *testing.T) {
|
||||
if stats.Counters.BackendMissesByRoute["api repos/:owner/:repo/actions/runs/:id/logs"] != 1 {
|
||||
t.Fatalf("backend misses by route = %#v", stats.Counters.BackendMissesByRoute)
|
||||
}
|
||||
if stats.Counters.BackendMissesByKey["api repos/openclaw/openclaw/actions/runs/123/logs -i"] != 1 {
|
||||
t.Fatalf("backend misses by key = %#v", stats.Counters.BackendMissesByKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGHShimXCacheStatsSinceAndSnapshot(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
configPath := seedGHShimRepo(t, ctx)
|
||||
dir := t.TempDir()
|
||||
ghPath := filepath.Join(dir, "gh")
|
||||
if err := os.WriteFile(ghPath, []byte("#!/bin/sh\necho repo:$*\n"), 0o755); err != nil {
|
||||
t.Fatalf("write fake gh: %v", err)
|
||||
}
|
||||
t.Setenv("GITCRAWL_GH_PATH", ghPath)
|
||||
t.Setenv("GH_REPO", "stats-since/"+filepath.Base(dir))
|
||||
t.Setenv("GITCRAWL_GH_CACHE_TTL", "1m")
|
||||
|
||||
run := New()
|
||||
var stdout bytes.Buffer
|
||||
run.Stdout = &stdout
|
||||
args := []string{"--config", configPath, "gh", "repo", "view", "openclaw/gitcrawl", "--json", "nameWithOwner"}
|
||||
if err := run.Run(ctx, args); err != nil {
|
||||
t.Fatalf("repo view: %v", err)
|
||||
}
|
||||
stdout.Reset()
|
||||
if err := run.Run(ctx, []string{"--config", configPath, "gh", "xcache", "stats", "--since", "1h", "--json"}); err != nil {
|
||||
t.Fatalf("xcache stats --since: %v", err)
|
||||
}
|
||||
var stats ghCommandCacheStats
|
||||
if err := json.Unmarshal(stdout.Bytes(), &stats); err != nil {
|
||||
t.Fatalf("decode stats: %v\n%s", err, stdout.String())
|
||||
}
|
||||
if stats.Since != "1h0m0s" || stats.CumulativeCounters == nil || stats.Counters.BackendMisses != 1 {
|
||||
t.Fatalf("since stats = %+v", stats)
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
if err := run.Run(ctx, []string{"--config", configPath, "gh", "xcache", "snapshot", "--reset", "--json"}); err != nil {
|
||||
t.Fatalf("xcache snapshot: %v", err)
|
||||
}
|
||||
var snap ghCommandCacheSnapshotResult
|
||||
if err := json.Unmarshal(stdout.Bytes(), &snap); err != nil {
|
||||
t.Fatalf("decode snapshot: %v\n%s", err, stdout.String())
|
||||
}
|
||||
if snap.SnapshotPath == "" || !snap.Reset {
|
||||
t.Fatalf("snapshot result = %+v", snap)
|
||||
}
|
||||
if _, err := os.Stat(snap.SnapshotPath); err != nil {
|
||||
t.Fatalf("snapshot file: %v", err)
|
||||
}
|
||||
stdout.Reset()
|
||||
if err := run.Run(ctx, []string{"--config", configPath, "gh", "xcache", "stats", "--json"}); err != nil {
|
||||
t.Fatalf("xcache stats after snapshot reset: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &stats); err != nil {
|
||||
t.Fatalf("decode reset stats: %v\n%s", err, stdout.String())
|
||||
}
|
||||
if stats.Counters.BackendMisses != 0 {
|
||||
t.Fatalf("snapshot reset counters = %+v", stats.Counters)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGHShimCachesReadOnlyFallbackErrors(t *testing.T) {
|
||||
@ -466,6 +555,80 @@ exit 1
|
||||
}
|
||||
}
|
||||
|
||||
func TestGHShimServesStaleWhileAnotherProcessRefreshes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
configPath := seedGHShimRepo(t, ctx)
|
||||
dir := t.TempDir()
|
||||
countPath := filepath.Join(dir, "count")
|
||||
ghPath := filepath.Join(dir, "gh")
|
||||
script := `#!/bin/sh
|
||||
count=0
|
||||
if [ -f "$GH_SHIM_COUNT" ]; then
|
||||
count=$(cat "$GH_SHIM_COUNT")
|
||||
fi
|
||||
count=$((count + 1))
|
||||
printf "%s" "$count" > "$GH_SHIM_COUNT"
|
||||
if [ "$count" != "1" ]; then
|
||||
sleep 1
|
||||
fi
|
||||
echo "release-$count"
|
||||
`
|
||||
if err := os.WriteFile(ghPath, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write fake gh: %v", err)
|
||||
}
|
||||
t.Setenv("GITCRAWL_GH_PATH", ghPath)
|
||||
t.Setenv("GH_SHIM_COUNT", countPath)
|
||||
t.Setenv("GITCRAWL_GH_CACHE_TTL", "1ns")
|
||||
t.Setenv("GITCRAWL_GH_STALE_GRACE", "1h")
|
||||
|
||||
args := []string{"--config", configPath, "gh", "release", "view", "v1", "-R", "openclaw/openclaw"}
|
||||
run := New()
|
||||
var stdout bytes.Buffer
|
||||
run.Stdout = &stdout
|
||||
if err := run.Run(ctx, args); err != nil {
|
||||
t.Fatalf("seed read: %v", err)
|
||||
}
|
||||
stdout.Reset()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
outputs := make(chan string, 2)
|
||||
errs := make(chan error, 2)
|
||||
for i := 0; i < 2; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
run := New()
|
||||
var out bytes.Buffer
|
||||
run.Stdout = &out
|
||||
if err := run.Run(ctx, args); err != nil {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
outputs <- strings.TrimSpace(out.String())
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
close(outputs)
|
||||
for err := range errs {
|
||||
t.Fatalf("stale while refresh run: %v", err)
|
||||
}
|
||||
seen := map[string]int{}
|
||||
for out := range outputs {
|
||||
seen[out]++
|
||||
}
|
||||
if seen["release-1"] != 1 || seen["release-2"] != 1 {
|
||||
t.Fatalf("outputs = %#v, want one stale and one refresh", seen)
|
||||
}
|
||||
countData, err := os.ReadFile(countPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read count: %v", err)
|
||||
}
|
||||
if strings.TrimSpace(string(countData)) != "2" {
|
||||
t.Fatalf("fake gh call count = %q, want 2", countData)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGHShimMutatingFallbackClearsMatchingCacheForGHXStyleMutations(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
configPath := seedGHShimRepo(t, ctx)
|
||||
|
||||
@ -347,6 +347,12 @@ func ghCompletedRunCacheTTL(entry ghCommandCacheEntry) time.Duration {
|
||||
}
|
||||
if entry.Args[0] == "api" {
|
||||
route := normalizeGHAPIRoute(entry.Args[1:])
|
||||
if strings.Contains(route, "/actions/runs/:id/jobs") && ghJSONJobsCompleted(entry.Stdout) {
|
||||
return 12 * time.Hour
|
||||
}
|
||||
if strings.Contains(route, "/actions/jobs/:id") && ghJSONStatusCompleted(entry.Stdout) {
|
||||
return 12 * time.Hour
|
||||
}
|
||||
if strings.Contains(route, "/actions/runs/:id") && ghJSONStatusCompleted(entry.Stdout) {
|
||||
return 12 * time.Hour
|
||||
}
|
||||
@ -357,6 +363,16 @@ func ghCompletedRunCacheTTL(entry ghCommandCacheEntry) time.Duration {
|
||||
return 0
|
||||
}
|
||||
|
||||
func ghJSONJobsCompleted(raw string) bool {
|
||||
var payload struct {
|
||||
Jobs []map[string]any `json:"jobs"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw), &payload); err == nil {
|
||||
return len(payload.Jobs) > 0 && allGHStatusMapsCompleted(payload.Jobs)
|
||||
}
|
||||
return ghJSONCollectionCompleted(raw)
|
||||
}
|
||||
|
||||
func ghJSONStatusCompleted(raw string) bool {
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal([]byte(raw), &payload); err != nil {
|
||||
|
||||
@ -4,17 +4,32 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ghXCacheCounters struct {
|
||||
LocalHits int64 `json:"local_hits"`
|
||||
FallbackHits int64 `json:"fallback_hits"`
|
||||
StaleHits int64 `json:"stale_hits"`
|
||||
BackendMisses int64 `json:"backend_misses"`
|
||||
PassThroughWrites int64 `json:"pass_through_writes"`
|
||||
LocalHits int64 `json:"local_hits"`
|
||||
FallbackHits int64 `json:"fallback_hits"`
|
||||
StaleHits int64 `json:"stale_hits"`
|
||||
BackendMisses int64 `json:"backend_misses"`
|
||||
PassThroughWrites int64 `json:"pass_through_writes"`
|
||||
BackendMissesByCommand map[string]int64 `json:"backend_misses_by_command,omitempty"`
|
||||
BackendMissesByRoute map[string]int64 `json:"backend_misses_by_route,omitempty"`
|
||||
BackendMissesByKey map[string]int64 `json:"backend_misses_by_key,omitempty"`
|
||||
Hourly map[string]ghXCacheCounterBucket `json:"hourly,omitempty"`
|
||||
}
|
||||
|
||||
type ghXCacheCounterBucket struct {
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
LocalHits int64 `json:"local_hits,omitempty"`
|
||||
FallbackHits int64 `json:"fallback_hits,omitempty"`
|
||||
StaleHits int64 `json:"stale_hits,omitempty"`
|
||||
BackendMisses int64 `json:"backend_misses,omitempty"`
|
||||
PassThroughWrites int64 `json:"pass_through_writes,omitempty"`
|
||||
BackendMissesByCommand map[string]int64 `json:"backend_misses_by_command,omitempty"`
|
||||
BackendMissesByRoute map[string]int64 `json:"backend_misses_by_route,omitempty"`
|
||||
BackendMissesByKey map[string]int64 `json:"backend_misses_by_key,omitempty"`
|
||||
}
|
||||
|
||||
func (a *App) ghXCacheCounters() (ghXCacheCounters, error) {
|
||||
@ -57,6 +72,28 @@ func (a *App) incrementGHXCacheCounterWithArgs(name string, args []string) error
|
||||
_ = os.Remove(lockPath)
|
||||
}()
|
||||
stats := readGHXCacheCounters(path)
|
||||
if !incrementGHXCacheCounters(&stats, name, args) {
|
||||
return nil
|
||||
}
|
||||
bucketKey, bucketStart := ghXCacheCurrentBucket(time.Now())
|
||||
if stats.Hourly == nil {
|
||||
stats.Hourly = map[string]ghXCacheCounterBucket{}
|
||||
}
|
||||
bucket := stats.Hourly[bucketKey]
|
||||
if bucket.StartedAt.IsZero() {
|
||||
bucket.StartedAt = bucketStart
|
||||
}
|
||||
_ = incrementGHXCacheCounterBucket(&bucket, name, args)
|
||||
stats.Hourly[bucketKey] = bucket
|
||||
pruneGHXCacheBuckets(stats.Hourly, time.Now().Add(-7*24*time.Hour))
|
||||
data, err := json.Marshal(stats)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeAtomicFile(path, data, 0o600)
|
||||
}
|
||||
|
||||
func incrementGHXCacheCounters(stats *ghXCacheCounters, name string, args []string) bool {
|
||||
switch name {
|
||||
case "local_hits":
|
||||
stats.LocalHits++
|
||||
@ -66,29 +103,69 @@ func (a *App) incrementGHXCacheCounterWithArgs(name string, args []string) error
|
||||
stats.StaleHits++
|
||||
case "backend_misses":
|
||||
stats.BackendMisses++
|
||||
if len(args) > 0 {
|
||||
if stats.BackendMissesByCommand == nil {
|
||||
stats.BackendMissesByCommand = map[string]int64{}
|
||||
}
|
||||
command := ghCommandName(args)
|
||||
stats.BackendMissesByCommand[command]++
|
||||
if route := ghCommandRoute(args); route != "" {
|
||||
if stats.BackendMissesByRoute == nil {
|
||||
stats.BackendMissesByRoute = map[string]int64{}
|
||||
}
|
||||
stats.BackendMissesByRoute[route]++
|
||||
}
|
||||
}
|
||||
incrementGHXCacheMissMaps(&stats.BackendMissesByCommand, &stats.BackendMissesByRoute, &stats.BackendMissesByKey, args)
|
||||
case "pass_through_writes":
|
||||
stats.PassThroughWrites++
|
||||
default:
|
||||
return nil
|
||||
return false
|
||||
}
|
||||
data, err := json.Marshal(stats)
|
||||
if err != nil {
|
||||
return err
|
||||
return true
|
||||
}
|
||||
|
||||
func incrementGHXCacheCounterBucket(bucket *ghXCacheCounterBucket, name string, args []string) bool {
|
||||
switch name {
|
||||
case "local_hits":
|
||||
bucket.LocalHits++
|
||||
case "fallback_hits":
|
||||
bucket.FallbackHits++
|
||||
case "stale_hits":
|
||||
bucket.StaleHits++
|
||||
case "backend_misses":
|
||||
bucket.BackendMisses++
|
||||
incrementGHXCacheMissMaps(&bucket.BackendMissesByCommand, &bucket.BackendMissesByRoute, &bucket.BackendMissesByKey, args)
|
||||
case "pass_through_writes":
|
||||
bucket.PassThroughWrites++
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return writeAtomicFile(path, data, 0o600)
|
||||
return true
|
||||
}
|
||||
|
||||
func incrementGHXCacheMissMaps(byCommand, byRoute, byKey *map[string]int64, args []string) {
|
||||
if len(args) == 0 {
|
||||
return
|
||||
}
|
||||
if *byCommand == nil {
|
||||
*byCommand = map[string]int64{}
|
||||
}
|
||||
(*byCommand)[ghCommandName(args)]++
|
||||
if route := ghCommandRoute(args); route != "" {
|
||||
if *byRoute == nil {
|
||||
*byRoute = map[string]int64{}
|
||||
}
|
||||
(*byRoute)[route]++
|
||||
}
|
||||
if key := ghCommandMissKey(args); key != "" {
|
||||
if *byKey == nil {
|
||||
*byKey = map[string]int64{}
|
||||
}
|
||||
(*byKey)[key]++
|
||||
}
|
||||
}
|
||||
|
||||
func ghCommandMissKey(args []string) string {
|
||||
if len(args) == 0 {
|
||||
return ""
|
||||
}
|
||||
canonical := canonicalGHCommandArgs(args)
|
||||
if len(canonical) == 0 {
|
||||
return ghCommandName(args)
|
||||
}
|
||||
key := strings.Join(canonical, " ")
|
||||
if len(key) > 180 {
|
||||
key = key[:177] + "..."
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
func ghCommandRoute(args []string) string {
|
||||
@ -116,6 +193,53 @@ func readGHXCacheCounters(path string) ghXCacheCounters {
|
||||
return stats
|
||||
}
|
||||
|
||||
func (c ghXCacheCounters) since(since time.Duration, now time.Time) ghXCacheCounters {
|
||||
if since <= 0 {
|
||||
return c
|
||||
}
|
||||
cutoff := now.Add(-since)
|
||||
var out ghXCacheCounters
|
||||
for _, bucket := range c.Hourly {
|
||||
if bucket.StartedAt.IsZero() || bucket.StartedAt.Before(cutoff) {
|
||||
continue
|
||||
}
|
||||
out.LocalHits += bucket.LocalHits
|
||||
out.FallbackHits += bucket.FallbackHits
|
||||
out.StaleHits += bucket.StaleHits
|
||||
out.BackendMisses += bucket.BackendMisses
|
||||
out.PassThroughWrites += bucket.PassThroughWrites
|
||||
mergeCounterMap(&out.BackendMissesByCommand, bucket.BackendMissesByCommand)
|
||||
mergeCounterMap(&out.BackendMissesByRoute, bucket.BackendMissesByRoute)
|
||||
mergeCounterMap(&out.BackendMissesByKey, bucket.BackendMissesByKey)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeCounterMap(dst *map[string]int64, src map[string]int64) {
|
||||
if len(src) == 0 {
|
||||
return
|
||||
}
|
||||
if *dst == nil {
|
||||
*dst = map[string]int64{}
|
||||
}
|
||||
for key, value := range src {
|
||||
(*dst)[key] += value
|
||||
}
|
||||
}
|
||||
|
||||
func ghXCacheCurrentBucket(now time.Time) (string, time.Time) {
|
||||
start := now.UTC().Truncate(time.Hour)
|
||||
return start.Format("2006-01-02T15:00:00Z"), start
|
||||
}
|
||||
|
||||
func pruneGHXCacheBuckets(buckets map[string]ghXCacheCounterBucket, cutoff time.Time) {
|
||||
for key, bucket := range buckets {
|
||||
if !bucket.StartedAt.IsZero() && bucket.StartedAt.Before(cutoff) {
|
||||
delete(buckets, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeAtomicFile(path string, data []byte, perm os.FileMode) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
|
||||
@ -13,16 +13,18 @@ import (
|
||||
)
|
||||
|
||||
type ghCommandCacheStats struct {
|
||||
CacheDir string `json:"cache_dir"`
|
||||
Entries int `json:"entries"`
|
||||
Expired int `json:"expired"`
|
||||
Locks int `json:"locks"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
CacheHits int64 `json:"cache_hits"`
|
||||
TotalReads int64 `json:"total_reads"`
|
||||
HitRatePercent float64 `json:"hit_rate_percent"`
|
||||
Counters ghXCacheCounters `json:"counters"`
|
||||
Commands map[string]ghCommandCacheCount `json:"commands"`
|
||||
CacheDir string `json:"cache_dir"`
|
||||
Entries int `json:"entries"`
|
||||
Expired int `json:"expired"`
|
||||
Locks int `json:"locks"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Since string `json:"since,omitempty"`
|
||||
CacheHits int64 `json:"cache_hits"`
|
||||
TotalReads int64 `json:"total_reads"`
|
||||
HitRatePercent float64 `json:"hit_rate_percent"`
|
||||
Counters ghXCacheCounters `json:"counters"`
|
||||
CumulativeCounters *ghXCacheCounters `json:"cumulative_counters,omitempty"`
|
||||
Commands map[string]ghCommandCacheCount `json:"commands"`
|
||||
}
|
||||
|
||||
type ghCommandCacheCount struct {
|
||||
@ -43,18 +45,28 @@ type ghCommandCacheKeyInfo struct {
|
||||
|
||||
func (a *App) runGHXCache(args []string) error {
|
||||
if len(args) == 0 {
|
||||
return usageErr(fmt.Errorf("usage: gh xcache <stats|keys|gc|flush|reset>"))
|
||||
return usageErr(fmt.Errorf("usage: gh xcache <stats|keys|gc|flush|reset|snapshot>"))
|
||||
}
|
||||
fs := flag.NewFlagSet("xcache "+args[0], flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
jsonOut := fs.Bool("json", false, "write JSON output")
|
||||
sinceRaw := fs.String("since", "", "show stats for the recent duration (stats only)")
|
||||
resetAfterSnapshot := fs.Bool("reset", false, "reset counters after writing a snapshot (snapshot only)")
|
||||
if err := fs.Parse(args[1:]); err != nil {
|
||||
return usageErr(err)
|
||||
}
|
||||
a.applyCommandJSON(*jsonOut)
|
||||
switch args[0] {
|
||||
case "stats":
|
||||
return a.runGHXCacheStats()
|
||||
var since time.Duration
|
||||
if strings.TrimSpace(*sinceRaw) != "" {
|
||||
parsed, err := time.ParseDuration(strings.TrimSpace(*sinceRaw))
|
||||
if err != nil || parsed <= 0 {
|
||||
return usageErr(fmt.Errorf("invalid --since duration %q", *sinceRaw))
|
||||
}
|
||||
since = parsed
|
||||
}
|
||||
return a.runGHXCacheStats(since)
|
||||
case "keys":
|
||||
return a.runGHXCacheKeys()
|
||||
case "gc":
|
||||
@ -63,13 +75,15 @@ func (a *App) runGHXCache(args []string) error {
|
||||
return a.runGHXCacheFlush()
|
||||
case "reset":
|
||||
return a.runGHXCacheReset()
|
||||
case "snapshot":
|
||||
return a.runGHXCacheSnapshot(*resetAfterSnapshot)
|
||||
default:
|
||||
return usageErr(fmt.Errorf("unknown xcache command %q", args[0]))
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) runGHXCacheStats() error {
|
||||
stats, err := a.ghCommandCacheStats()
|
||||
func (a *App) runGHXCacheStats(since time.Duration) error {
|
||||
stats, err := a.ghCommandCacheStats(since)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -87,11 +101,15 @@ func (a *App) runGHXCacheStats() error {
|
||||
_, _ = fmt.Fprintf(a.Stdout, " %-16s %d entries / %d bytes\n", command, count.Entries, count.Bytes)
|
||||
}
|
||||
}
|
||||
if stats.Since != "" {
|
||||
_, _ = fmt.Fprintf(a.Stdout, "\nSince: %s\n", stats.Since)
|
||||
}
|
||||
_, _ = fmt.Fprintf(a.Stdout, "\nCounters:\n local hits: %d\n fallback hits: %d\n stale hits: %d\n backend misses: %d\n pass-through writes: %d\n hit rate: %.1f%% (%d/%d reads)\n",
|
||||
stats.Counters.LocalHits, stats.Counters.FallbackHits, stats.Counters.StaleHits, stats.Counters.BackendMisses, stats.Counters.PassThroughWrites,
|
||||
stats.HitRatePercent, stats.CacheHits, stats.TotalReads)
|
||||
printGHXCacheMisses(a.Stdout, "Backend Misses by Command", stats.Counters.BackendMissesByCommand)
|
||||
printGHXCacheMisses(a.Stdout, "Backend Misses by Route", stats.Counters.BackendMissesByRoute)
|
||||
printGHXCacheMisses(a.Stdout, "Backend Misses by Key", stats.Counters.BackendMissesByKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -161,6 +179,48 @@ func (a *App) runGHXCacheReset() error {
|
||||
return err
|
||||
}
|
||||
|
||||
type ghCommandCacheSnapshotResult struct {
|
||||
SnapshotPath string `json:"snapshot_path"`
|
||||
Reset bool `json:"reset"`
|
||||
}
|
||||
|
||||
func (a *App) runGHXCacheSnapshot(reset bool) error {
|
||||
stats, err := a.ghCommandCacheStats(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir, err := a.ghCommandCacheDir()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
snapshotDir := filepath.Join(dir, "_snapshots")
|
||||
if err := os.MkdirAll(snapshotDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
path := filepath.Join(snapshotDir, time.Now().UTC().Format("20060102T150405Z")+".json")
|
||||
data, err := json.MarshalIndent(stats, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeAtomicFile(path, data, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if reset {
|
||||
if err := a.resetGHXCacheCounters(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
result := ghCommandCacheSnapshotResult{SnapshotPath: path, Reset: reset}
|
||||
if a.format == FormatJSON {
|
||||
return a.writeJSONValue(result, "")
|
||||
}
|
||||
_, err = fmt.Fprintf(a.Stdout, "Wrote xcache snapshot: %s\n", path)
|
||||
if err == nil && reset {
|
||||
_, err = fmt.Fprintln(a.Stdout, "Reset xcache counters")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type ghCommandCacheGCResult struct {
|
||||
Removed int `json:"removed"`
|
||||
LocksRemoved int `json:"locks_removed"`
|
||||
@ -178,7 +238,7 @@ func (a *App) runGHXCacheGC() error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *App) ghCommandCacheStats() (ghCommandCacheStats, error) {
|
||||
func (a *App) ghCommandCacheStats(since time.Duration) (ghCommandCacheStats, error) {
|
||||
dir, err := a.ghCommandCacheDir()
|
||||
if err != nil {
|
||||
return ghCommandCacheStats{}, err
|
||||
@ -188,9 +248,15 @@ func (a *App) ghCommandCacheStats() (ghCommandCacheStats, error) {
|
||||
return ghCommandCacheStats{}, err
|
||||
}
|
||||
counters, _ := a.ghXCacheCounters()
|
||||
cumulative := counters
|
||||
stats := ghCommandCacheStats{CacheDir: dir, Locks: locks, Counters: counters, Commands: map[string]ghCommandCacheCount{}}
|
||||
stats.CacheHits = counters.LocalHits + counters.FallbackHits + counters.StaleHits
|
||||
stats.TotalReads = stats.CacheHits + counters.BackendMisses
|
||||
if since > 0 {
|
||||
stats.Since = since.String()
|
||||
stats.CumulativeCounters = &cumulative
|
||||
stats.Counters = counters.since(since, time.Now())
|
||||
}
|
||||
stats.CacheHits = stats.Counters.LocalHits + stats.Counters.FallbackHits + stats.Counters.StaleHits
|
||||
stats.TotalReads = stats.CacheHits + stats.Counters.BackendMisses
|
||||
if stats.TotalReads > 0 {
|
||||
stats.HitRatePercent = float64(stats.CacheHits) / float64(stats.TotalReads) * 100
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user