Revert "fix: keep WSL2 sync on fast rsync path"

This reverts commit d6ee5620dc.
This commit is contained in:
Vincent Koc 2026-05-04 23:32:54 -07:00
parent ac4c1953f0
commit 273dbfa0f5
No known key found for this signature in database
8 changed files with 138 additions and 189 deletions

View File

@ -6,9 +6,8 @@
- Fixed managed AWS Windows WSL2 bootstrap by using the current Ubuntu WSL rootfs URL, downloading large rootfs files through `curl.exe`, and retrying empty or partial rootfs downloads instead of reusing a poisoned tarball. Thanks @vincentkoc.
- Fixed AWS Windows WSL2 mode overrides so they refresh the default instance type to a nested-virtualization-capable family. Thanks @steipete.
- Fixed AWS Windows WSL2 runs so mode overrides also refresh the default work root to `/work/crabbox` while keeping WSL2 sync on the fast rsync path.
- Fixed AWS Windows WSL2 runs so mode overrides also refresh the default work root to `/work/crabbox` and sync via a WSL archive stream instead of rsync's remote protocol through Windows OpenSSH.
- Fixed remote git seeding so an unfetchable local commit cannot leave an empty `.git` worktree that makes sync sanity report every tracked file as deleted.
- Skipped remote git seeding for local commits that are not present in any remote-tracking ref, avoiding slow doomed clone/fetch attempts before rsync.
- Fixed Windows archive sync from macOS so Apple extended attributes do not spam remote tar warnings.
## 0.5.0 - 2026-05-04

View File

@ -113,13 +113,6 @@ func gitOutput(root string, args ...string) string {
return strings.TrimSpace(string(out))
}
func remoteGitSeedCandidate(repo Repo) bool {
if repo.Root == "" || repo.RemoteURL == "" || repo.Head == "" {
return false
}
return gitOutput(repo.Root, "for-each-ref", "--contains", repo.Head, "--format=%(refname)", "refs/remotes") != ""
}
func defaultBaseRef(root string) string {
originHead := gitOutput(root, "symbolic-ref", "--quiet", "--short", "refs/remotes/origin/HEAD")
if originHead != "" {

View File

@ -140,26 +140,6 @@ func TestSyncManifestDoesNotDeleteRecreatedStagedDelete(t *testing.T) {
}
}
func TestRemoteGitSeedCandidateRequiresRemoteTrackingRef(t *testing.T) {
dir := t.TempDir()
runGit(t, dir, "init")
runGit(t, dir, "config", "user.email", "test@example.com")
runGit(t, dir, "config", "user.name", "Test")
writeFile(t, filepath.Join(dir, "foo.txt"), "old")
runGit(t, dir, "add", ".")
runGit(t, dir, "commit", "-m", "init")
head := gitOutput(dir, "rev-parse", "HEAD")
repo := Repo{Root: dir, RemoteURL: "https://github.com/openclaw/crabbox.git", Head: head}
if remoteGitSeedCandidate(repo) {
t.Fatal("unpublished head should not be a seed candidate")
}
runGit(t, dir, "update-ref", "refs/remotes/origin/main", head)
if !remoteGitSeedCandidate(repo) {
t.Fatal("head in a remote-tracking ref should be a seed candidate")
}
}
func TestCheckSyncPreflightFailsLargeCandidate(t *testing.T) {
cfg := baseConfig()
cfg.Sync.FailFiles = 2

View File

@ -425,13 +425,15 @@ func (a App) runCommand(ctx context.Context, args []string) (err error) {
}
recorder.Event("sync.started", "sync", "")
timings.syncSteps.sshReady = time.Since(stepStart)
stepStart = time.Now()
mkdirCommand := remoteMkdir(workdir)
if isWindowsNativeTarget(target) {
stepStart = time.Now()
if err := runSSHQuiet(ctx, target, windowsRemoteMkdir(workdir)); err != nil {
return recordFailure(exit(7, "create remote workdir: %v", err))
}
timings.syncSteps.mkdir = time.Since(stepStart)
mkdirCommand = windowsRemoteMkdir(workdir)
}
if err := runSSHQuiet(ctx, target, mkdirCommand); err != nil {
return recordFailure(exit(7, "create remote workdir: %v", err))
}
timings.syncSteps.mkdir = time.Since(stepStart)
stepStart = time.Now()
manifest, err := syncManifest(repo.Root, configuredExcludes(cfg))
if err != nil {
@ -454,6 +456,17 @@ func (a App) runCommand(ctx context.Context, args []string) (err error) {
recorder.Event("sync.finished", "synced", fmt.Sprintf("duration=%s mode=archive", timings.sync.Round(time.Millisecond)))
goto afterSync
}
if isWindowsWSL2Target(target) {
stepStart = time.Now()
if err := syncWindowsWSL2(ctx, target, repo, cfg, workdir, manifest, a.Stdout, a.Stderr, rsyncOptions{Debug: *debugSync, Delete: cfg.Sync.Delete, Checksum: cfg.Sync.Checksum, Timeout: cfg.Sync.Timeout, HeartbeatInterval: 15 * time.Second}); err != nil {
return recordFailure(err)
}
timings.syncSteps.rsync = time.Since(stepStart)
timings.sync = time.Since(syncStart)
fmt.Fprintf(a.Stderr, "sync complete in %s\n", timings.sync.Round(time.Millisecond))
recorder.Event("sync.finished", "synced", fmt.Sprintf("duration=%s mode=archive-wsl2", timings.sync.Round(time.Millisecond)))
goto afterSync
}
fingerprint := ""
if cfg.Sync.Fingerprint {
stepStart = time.Now()
@ -474,7 +487,7 @@ func (a App) runCommand(ctx context.Context, args []string) (err error) {
}
}
}
if cfg.Sync.GitSeed && remoteGitSeedCandidate(repo) {
if cfg.Sync.GitSeed {
stepStart = time.Now()
if err := runSSHQuiet(ctx, target, remoteGitSeed(workdir, repo.RemoteURL, repo.Head)); err != nil {
fmt.Fprintf(a.Stderr, "warning: remote git seed failed: %v\n", err)
@ -483,9 +496,11 @@ func (a App) runCommand(ctx context.Context, args []string) (err error) {
}
manifestData := manifest.NUL()
stepStart = time.Now()
manifestInput := fmt.Sprintf("%d\n", len(manifestData)) + string(manifestData) + string(manifest.DeletedNUL())
if err := runSSHInputQuiet(ctx, target, remoteWriteSyncManifestsNew(workdir), manifestInput); err != nil {
return recordFailure(exit(7, "write sync manifests: %v", err))
if err := runSSHInputQuiet(ctx, target, remoteWriteSyncManifestNew(workdir), string(manifestData)); err != nil {
return recordFailure(exit(7, "write sync manifest: %v", err))
}
if err := runSSHInputQuiet(ctx, target, remoteWriteSyncDeletedNew(workdir), string(manifest.DeletedNUL())); err != nil {
return recordFailure(exit(7, "write sync delete manifest: %v", err))
}
timings.syncSteps.manifestWrite = time.Since(stepStart)
if cfg.Sync.Delete {
@ -500,33 +515,49 @@ func (a App) runCommand(ctx context.Context, args []string) (err error) {
return recordFailure(exit(6, "rsync failed: %v", err))
}
timings.syncSteps.rsync = time.Since(stepStart)
stepStart = time.Now()
if err := runSSHQuiet(ctx, target, remoteApplySyncManifest(workdir)); err != nil {
return recordFailure(exit(6, "remote sync manifest apply failed: %v", err))
}
timings.syncSteps.manifestApply = time.Since(stepStart)
stepStart = time.Now()
if out, err := runSSHCombinedOutput(ctx, target, remoteSyncSanity(workdir, os.Getenv("CRABBOX_ALLOW_MASS_DELETIONS") == "1")); err != nil {
if out != "" {
return recordFailure(exit(6, "remote sync sanity failed: %s: %v", out, err))
}
return recordFailure(exit(6, "remote sync sanity failed: %v", err))
}
timings.syncSteps.sanity = time.Since(stepStart)
baseSHA := gitHydrateBaseSHA(repo, cfg.Sync.BaseRef)
hydrateGit := true
if hydratedByActions {
stepStart = time.Now()
reason, err := runSSHOutput(ctx, target, remoteGitHydrateStatus(workdir, cfg.Sync.BaseRef, baseSHA))
if err == nil && reason != "" {
timings.syncSteps.gitHydrateSkipped = true
timings.syncSteps.gitHydrateSkipReason = reason
hydrateGit = false
fmt.Fprintf(a.Stderr, "skipping git hydrate: %s\n", reason)
}
}
stepStart = time.Now()
finalizeCommand := remoteFinalizeSync(workdir, remoteSyncFinalizeOptions{
AllowMassDeletions: os.Getenv("CRABBOX_ALLOW_MASS_DELETIONS") == "1",
HydrateGit: hydrateGit,
BaseRef: cfg.Sync.BaseRef,
BaseSHA: baseSHA,
Fingerprint: fingerprint,
})
if out, err := runSSHCombinedOutput(ctx, target, finalizeCommand); err != nil {
if out != "" {
return recordFailure(exit(6, "remote sync finalize failed: %s: %v", out, err))
if !timings.syncSteps.gitHydrateSkipped {
stepStart = time.Now()
if err := runSSHQuiet(ctx, target, remoteGitHydrate(workdir, cfg.Sync.BaseRef)); err != nil {
fmt.Fprintf(a.Stderr, "warning: remote git hydrate failed: %v\n", err)
}
return recordFailure(exit(6, "remote sync finalize failed: %v", err))
timings.syncSteps.gitHydrate = time.Since(stepStart)
}
if cfg.Sync.BaseRef != "" && baseSHA != "" {
stepStart = time.Now()
if err := runSSHQuiet(ctx, target, remoteWriteGitHydrateMarker(workdir, cfg.Sync.BaseRef, baseSHA)); err != nil {
fmt.Fprintf(a.Stderr, "warning: write git hydrate marker failed: %v\n", err)
}
}
if fingerprint != "" {
stepStart = time.Now()
if err := runSSHQuiet(ctx, target, remoteWriteSyncFingerprint(workdir, fingerprint)); err != nil {
fmt.Fprintf(a.Stderr, "warning: write sync fingerprint failed: %v\n", err)
}
timings.syncSteps.fingerprintWrite = time.Since(stepStart)
}
timings.syncSteps.finalize = time.Since(stepStart)
timings.sync = time.Since(syncStart)
fmt.Fprintf(a.Stderr, "sync complete in %s\n", timings.sync.Round(time.Millisecond))
recorder.Event("sync.finished", "synced", fmt.Sprintf("duration=%s skipped=false", timings.sync.Round(time.Millisecond)))
@ -655,7 +686,6 @@ type syncStepTimings struct {
manifestApply time.Duration
sanity time.Duration
gitHydrate time.Duration
finalize time.Duration
gitHydrateSkipped bool
gitHydrateSkipReason string
fingerprintWrite time.Duration
@ -699,7 +729,6 @@ func formatSyncStepTimings(steps syncStepTimings) string {
} else {
appendStep("git_hydrate", steps.gitHydrate)
}
appendStep("finalize", steps.finalize)
appendStep("fingerprint_write", steps.fingerprintWrite)
return strings.Join(parts, ",")
}

View File

@ -592,14 +592,6 @@ func remoteWriteSyncFingerprint(workdir, fingerprint string) string {
return "bash -lc " + shellQuote(script)
}
type remoteSyncFinalizeOptions struct {
AllowMassDeletions bool
HydrateGit bool
BaseRef string
BaseSHA string
Fingerprint string
}
func remoteWriteSyncManifestNew(workdir string) string {
script := "cd " + shellQuote(workdir) + " && " + remoteSyncMetaDirScript() + "mkdir -p \"$meta_dir\" && cat > \"$meta_dir/sync-manifest.new\""
return "bash -lc " + shellQuote(script)
@ -610,20 +602,6 @@ func remoteWriteSyncDeletedNew(workdir string) string {
return "bash -lc " + shellQuote(script)
}
func remoteWriteSyncManifestsNew(workdir string) string {
python := `import pathlib
import sys
manifest_len = int(sys.stdin.buffer.readline())
manifest = sys.stdin.buffer.read(manifest_len)
deleted = sys.stdin.buffer.read()
pathlib.Path(sys.argv[1]).write_bytes(manifest)
pathlib.Path(sys.argv[2]).write_bytes(deleted)
`
script := "mkdir -p " + shellQuote(workdir) + " && cd " + shellQuote(workdir) + " && " + remoteSyncMetaDirScript() + "mkdir -p \"$meta_dir\" && python3 -c " + shellQuote(python) + " \"$meta_dir/sync-manifest.new\" \"$meta_dir/sync-deleted.new\""
return "bash -lc " + shellQuote(script)
}
func remotePruneSyncManifest(workdir string) string {
script := "set -e\ncd " + shellQuote(workdir) + `
` + remoteSyncMetaDirScript() + `
@ -669,46 +647,6 @@ func remoteApplySyncManifest(workdir string) string {
return "bash -lc " + shellQuote(script)
}
func remoteFinalizeSync(workdir string, opts remoteSyncFinalizeOptions) string {
allowValue := ""
if opts.AllowMassDeletions {
allowValue = "1"
}
script := `set -e
cd ` + shellQuote(workdir) + `
` + remoteSyncMetaDirScript() + `
mkdir -p "$meta_dir"
new="$meta_dir/sync-manifest.new"
deleted="$meta_dir/sync-deleted.new"
rm -f "$deleted"
mv "$new" "$meta_dir/sync-manifest"
if test -d .git && git status --short >/tmp/crabbox-git-status 2>/dev/null; then
deletions=$(awk '/^ D|^D / { n++ } END { print n+0 }' /tmp/crabbox-git-status)
if [ ` + shellQuote(allowValue) + ` != '1' ] && [ "$deletions" -ge 200 ]; then
echo "remote sync sanity failed: $deletions tracked deletions" >&2
awk '/^ D|^D / { print " " substr($0,4) }' /tmp/crabbox-git-status | head -20 >&2
exit 66
fi
fi
`
if opts.HydrateGit && opts.BaseRef != "" {
refspec := "+refs/heads/" + opts.BaseRef + ":refs/remotes/origin/" + opts.BaseRef
script += `if git rev-parse --is-inside-work-tree >/dev/null 2>&1 && git remote get-url origin >/dev/null 2>&1; then
git fetch --quiet --unshallow origin ` + shellQuote(refspec) + ` || git fetch --quiet --depth=1000 origin ` + shellQuote(refspec) + ` || git fetch --quiet origin ` + shellQuote(refspec) + ` || git fetch --quiet origin ` + shellQuote(opts.BaseRef) + ` || true
fi
`
}
if opts.BaseRef != "" && opts.BaseSHA != "" {
script += `printf %s ` + shellQuote(opts.BaseRef+" "+opts.BaseSHA+"\n") + ` > "$meta_dir/git-hydrate-base" || true
`
}
if opts.Fingerprint != "" {
script += `printf %s ` + shellQuote(opts.Fingerprint) + ` > "$meta_dir/sync-fingerprint" || true
`
}
return "bash -lc " + shellQuote(script)
}
func remoteSyncMetaDirScript() string {
return "meta_dir=$(if [ -d .git ]; then printf %s .git/crabbox; else printf %s .crabbox; fi); "
}

View File

@ -4,7 +4,6 @@ import (
"bytes"
"context"
"encoding/base64"
"fmt"
"os"
"os/exec"
"path/filepath"
@ -372,53 +371,23 @@ func TestRemoteApplySyncManifestOnlyCommitsManifest(t *testing.T) {
}
}
func TestRemoteFinalizeSyncCommitsMetadataInOneCommand(t *testing.T) {
func TestPOSIXPrepareWorkdirDeletesButPreservesGit(t *testing.T) {
workdir := t.TempDir()
if err := os.Mkdir(filepath.Join(workdir, ".git"), 0o755); err != nil {
t.Fatal(err)
}
metaDir := filepath.Join(workdir, ".git", "crabbox")
if err := os.MkdirAll(metaDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(metaDir, "sync-manifest.new"), []byte("tracked.txt\x00"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(metaDir, "sync-deleted.new"), []byte("deleted.txt\x00"), 0o644); err != nil {
t.Fatal(err)
}
mustWriteTestFile(t, filepath.Join(workdir, ".git", "config"), "git")
mustWriteTestFile(t, filepath.Join(workdir, "stale.txt"), "stale")
mustWriteTestFile(t, filepath.Join(workdir, "old", "nested.txt"), "old")
cmd := exec.Command("bash", "-lc", remoteFinalizeSync(workdir, remoteSyncFinalizeOptions{
BaseRef: "main",
BaseSHA: "abc123",
Fingerprint: "fp123",
}))
cmd := exec.Command("bash", "-lc", posixPrepareWorkdir(workdir, true))
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("remote finalize failed: %v\n%s", err, out)
t.Fatalf("posix prepare failed: %v\n%s", err, out)
}
if _, err := os.Stat(filepath.Join(metaDir, "sync-deleted.new")); !os.IsNotExist(err) {
t.Fatalf("deleted manifest should be removed, stat err=%v", err)
if _, err := os.Stat(filepath.Join(workdir, ".git", "config")); err != nil {
t.Fatalf(".git should survive prepare: %v", err)
}
manifest, err := os.ReadFile(filepath.Join(metaDir, "sync-manifest"))
if err != nil {
t.Fatal(err)
}
if string(manifest) != "tracked.txt\x00" {
t.Fatalf("unexpected manifest: %q", manifest)
}
marker, err := os.ReadFile(filepath.Join(metaDir, "git-hydrate-base"))
if err != nil {
t.Fatal(err)
}
if string(marker) != "main abc123\n" {
t.Fatalf("unexpected hydrate marker: %q", marker)
}
fingerprint, err := os.ReadFile(filepath.Join(metaDir, "sync-fingerprint"))
if err != nil {
t.Fatal(err)
}
if string(fingerprint) != "fp123" {
t.Fatalf("unexpected fingerprint: %q", fingerprint)
for _, rel := range []string{"stale.txt", "old"} {
if _, err := os.Stat(filepath.Join(workdir, rel)); !os.IsNotExist(err) {
t.Fatalf("%s should be removed, stat err=%v", rel, err)
}
}
}
@ -469,33 +438,6 @@ func TestRemoteWriteSyncDeletedNew(t *testing.T) {
}
}
func TestRemoteWriteSyncManifestsNew(t *testing.T) {
workdir := t.TempDir()
manifest := "keep.txt\x00"
deleted := "old.txt\x00"
input := fmt.Sprintf("%d\n", len(manifest)) + manifest + deleted
cmd := exec.Command("bash", "-lc", remoteWriteSyncManifestsNew(workdir))
cmd.Stdin = strings.NewReader(input)
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("write manifests failed: %v\n%s", err, out)
}
metaDir := filepath.Join(workdir, ".crabbox")
gotManifest, err := os.ReadFile(filepath.Join(metaDir, "sync-manifest.new"))
if err != nil {
t.Fatal(err)
}
if string(gotManifest) != manifest {
t.Fatalf("unexpected manifest: %q", gotManifest)
}
gotDeleted, err := os.ReadFile(filepath.Join(metaDir, "sync-deleted.new"))
if err != nil {
t.Fatal(err)
}
if string(gotDeleted) != deleted {
t.Fatalf("unexpected deleted manifest: %q", gotDeleted)
}
}
func TestRemoteSyncMetadataUsesGitDirForGitWorktree(t *testing.T) {
workdir := t.TempDir()
if err := os.Mkdir(filepath.Join(workdir, ".git"), 0o755); err != nil {

View File

@ -13,7 +13,7 @@ func syncWindowsNative(ctx context.Context, target SSHTarget, repo Repo, cfg Con
if err := runSSHQuiet(ctx, target, windowsPrepareWorkdir(workdir, cfg.Sync.Delete)); err != nil {
return exit(7, "prepare remote workdir: %v", err)
}
if cfg.Sync.GitSeed && remoteGitSeedCandidate(repo) {
if cfg.Sync.GitSeed {
if err := runSSHQuiet(ctx, target, windowsGitSeed(workdir, repo.RemoteURL, repo.Head)); err != nil {
fmt.Fprintf(stderr, "warning: remote git seed failed: %v\n", err)
}
@ -47,10 +47,79 @@ func syncWindowsNative(ctx context.Context, target SSHTarget, repo Repo, cfg Con
return nil
}
func syncWindowsWSL2(ctx context.Context, target SSHTarget, repo Repo, cfg Config, workdir string, manifest SyncManifest, stdout, stderr anyWriter, opts rsyncOptions) error {
if err := runSSHQuiet(ctx, target, posixPrepareWorkdir(workdir, cfg.Sync.Delete)); err != nil {
return exit(7, "prepare remote workdir: %v", err)
}
if cfg.Sync.GitSeed {
if err := runSSHQuiet(ctx, target, remoteGitSeed(workdir, repo.RemoteURL, repo.Head)); err != nil {
fmt.Fprintf(stderr, "warning: remote git seed failed: %v\n", err)
}
}
if err := syncArchive(ctx, target, repo, workdir, manifest, stdout, stderr, opts); err != nil {
return err
}
if out, err := runSSHCombinedOutput(ctx, target, remoteSyncSanity(workdir, false)); err != nil {
if out != "" {
return exit(6, "remote sync sanity failed: %s: %v", out, err)
}
return exit(6, "remote sync sanity failed: %v", err)
}
return nil
}
func syncArchive(ctx context.Context, target SSHTarget, repo Repo, workdir string, manifest SyncManifest, stdout, stderr anyWriter, opts rsyncOptions) error {
var input bytes.Buffer
input.Write(manifest.NUL())
cmd := exec.CommandContext(ctx, "tar", "-czf", "-", "-C", repo.Root, "--null", "-T", "-")
cmd.Stdin = &input
cmd.Env = append(os.Environ(), "COPYFILE_DISABLE=1")
var archive bytes.Buffer
cmd.Stdout = &archive
cmd.Stderr = stderr
start := time.Now()
if err := cmd.Run(); err != nil {
return exit(6, "create sync archive: %v", err)
}
if opts.Timeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, opts.Timeout)
defer cancel()
}
stopHeartbeat := startSyncHeartbeat(stderr, start, opts.HeartbeatInterval)
err := runSSHInput(ctx, target, posixExtractArchive(workdir), &archive, stdout, stderr)
stopHeartbeat()
if ctx.Err() == context.DeadlineExceeded {
return exit(6, "archive sync timed out after %s", opts.Timeout)
}
if err != nil {
return exit(6, "archive sync failed: %v", err)
}
return nil
}
type anyWriter interface {
Write([]byte) (int, error)
}
func posixPrepareWorkdir(workdir string, delete bool) string {
deleteScript := ""
if delete {
deleteScript = `
find . -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf -- {} +
`
}
return "bash -lc " + shellQuote(`set -e
mkdir -p `+shellQuote(workdir)+`
cd `+shellQuote(workdir)+deleteScript)
}
func posixExtractArchive(workdir string) string {
return "bash -lc " + shellQuote(`set -e
mkdir -p `+shellQuote(workdir)+`
tar -xzf - -C `+shellQuote(workdir))
}
func windowsPrepareWorkdir(workdir string, delete bool) string {
deleteScript := ""
if delete {

View File

@ -76,7 +76,6 @@ func syncTimingPhases(steps syncStepTimings) []timingPhase {
if steps.gitHydrateSkipped {
phases = append(phases, timingPhase{Name: "git_hydrate", Skipped: true, Reason: steps.gitHydrateSkipReason})
}
appendDuration("finalize", steps.finalize)
appendDuration("fingerprint_write", steps.fingerprintWrite)
return phases
}