diff --git a/CHANGELOG.md b/CHANGELOG.md index cc6b5bf..877b508 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## 0.14.0 - Unreleased ### Added -- Backup: add `gog backup` with age-encrypted Git shards, Gmail labels/raw message export, manifest status, full decrypt-and-verify, shard `cat`, local plaintext export, docs, and security-focused regression coverage. +- Backup: add `gog backup` with age-encrypted Git shards, Gmail labels/raw message export, Calendar/Contacts/Tasks/Drive metadata adapters, manifest status, full decrypt-and-verify, shard `cat`, local plaintext export, docs, and security-focused regression coverage. ### Fixed - Drive: include `driveId` in `drive ls`, `drive search`, and `drive get` field masks so Shared Drive files can be identified in JSON output. (#524) — thanks @LeanSheng. diff --git a/README.md b/README.md index a6fedc6..8088cce 100644 --- a/README.md +++ b/README.md @@ -725,7 +725,7 @@ Gmail watch (Pub/Sub push): ```bash gog backup init --repo ~/Projects/backup-gog --remote https://github.com/steipete/backup-gog.git -gog backup push --services gmail --account you@gmail.com +gog backup push --services all --account you@gmail.com gog backup status gog backup verify gog backup cat data/gmail//labels.jsonl.gz.age --pretty @@ -743,9 +743,15 @@ private age identity locally at `~/.gog/age.key`; GitHub only receives public `age1...` recipients, `manifest.json`, and encrypted `*.jsonl.gz.age` payloads. The private `AGE-SECRET-KEY-...` value must stay local or in a password manager. +Supported backup services are `gmail`, `calendar`, `contacts`, `tasks`, and +`drive` metadata; `all` expands to those services. A service-specific push +updates that service and preserves existing shards for services that were not +selected, as long as recipients are unchanged. + Use `gog backup cat` to decrypt one shard as JSONL, or `gog backup export` to write a local plaintext copy. The export writes Gmail messages as `.eml` files, plus `gmail//messages/index.jsonl` and pretty `labels.json`. +Other services export as verified JSONL under `raw/`. That export is intentionally unencrypted; keep it out of Git, shared folders, and cloud sync unless that is intentional. diff --git a/docs/backup.md b/docs/backup.md index 682e736..57e765b 100644 --- a/docs/backup.md +++ b/docs/backup.md @@ -24,7 +24,13 @@ gog backup init \ --remote https://github.com/steipete/backup-gog.git ``` -Back up Gmail: +Back up all supported services: + +```bash +gog backup push --services all --account steipete@gmail.com +``` + +Back up only Gmail: ```bash gog backup push --services gmail --account steipete@gmail.com @@ -63,6 +69,20 @@ gog backup export --out ~/Documents/gog-backup-export Use `--no-push` on `init` or `push` to commit locally without pushing to the remote. +Supported services: + +- `gmail`: labels and raw MIME messages. +- `calendar`: calendar list entries and all events, including deleted events. +- `contacts`: People API contacts and other contacts. +- `tasks`: task lists and tasks, including completed, deleted, hidden, and + assigned tasks. +- `drive`: shared drives and Drive file metadata. File contents are not copied + by the Drive adapter yet. + +`all` expands to every supported service. Pushing a subset updates that subset +and preserves existing shards for services that were not selected, as long as +the age recipients are unchanged. + ## Files Local config: @@ -79,6 +99,10 @@ README.md manifest.json data/gmail//labels.jsonl.gz.age data/gmail//messages/YYYY/MM/part-0001.jsonl.gz.age +data/calendar//... +data/contacts//... +data/drive//... +data/tasks//... ``` `manifest.json` is intentionally cleartext. It contains format version, export @@ -100,9 +124,9 @@ raw//... `gog backup export` decrypts and verifies the manifest-backed shards before writing files. Gmail messages become `.eml` files that open in Mail and other -mail clients. The export is not encrypted; do not place it inside the backup -Git repository, and keep it out of synced/shared folders unless that is -intentional. +mail clients. Other services are written as verified JSONL under `raw/`. The +export is not encrypted; do not place it inside the backup Git repository, and +keep it out of synced/shared folders unless that is intentional. ## Encryption @@ -175,9 +199,9 @@ Future hardening ideas: - Add optional shard padding or disable gzip for deployments that care more about size side channels than repository size. -## Gmail Adapter +## Service Adapters -The first adapter backs up: +The Gmail adapter backs up: - Gmail labels. - Raw Gmail messages from `users.messages.get(format=raw)`. @@ -189,6 +213,14 @@ friendly. `--include-spam-trash` defaults to true. Use `--query` and `--max` for bounded test exports; omit them for a full mailbox scan. +The Calendar adapter backs up calendar list entries and all events from each +calendar. The Contacts adapter backs up contacts and other contacts. The Tasks +adapter backs up task lists and tasks. The Drive adapter backs up shared drives +and file metadata, including names, owners, parents, links, checksums, export +links, and selected custom properties. Drive file contents are intentionally +left for a later adapter pass because they need format choices, bandwidth +limits, and resume/checkpoint behavior. + ## Adding Services Keep one backup engine and add service adapters. A service adapter should: @@ -200,4 +232,5 @@ Keep one backup engine and add service adapters. A service adapter should: 4. Add cleartext manifest counts and account hashes only. 5. Support bounded smoke flags when the service can be huge. -Good next adapters: Calendar, Contacts/People, Tasks, then Drive. +Good next adapters: Drive file content export, Docs/Sheets/Slides native +exports, Chat, Forms, Classroom, and Apps Script. diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 678d295..2ccd627 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -197,7 +197,15 @@ func NewJSONLShard(service, kind, account, rel string, rows any) (PlainShard, er func writeSnapshot(ctx context.Context, cfg Config, snapshot Snapshot, old Manifest) (Manifest, error) { recipients := normalizedStrings(cfg.Recipients) reuseEncrypted := sameStrings(old.Recipients, recipients) - shards := make([]ShardEntry, 0, len(snapshot.Shards)) + updatedServices := snapshotServices(snapshot) + shards := make([]ShardEntry, 0, len(old.Shards)+len(snapshot.Shards)) + if reuseEncrypted { + for _, shard := range old.Shards { + if _, ok := updatedServices[shard.Service]; !ok { + shards = append(shards, shard) + } + } + } for _, shard := range snapshot.Shards { select { case <-ctx.Done(): @@ -217,9 +225,9 @@ func writeSnapshot(ctx context.Context, cfg Config, snapshot Snapshot, old Manif Encrypted: true, Exported: time.Now().UTC(), Recipients: recipients, - Services: normalizedStrings(snapshot.Services), - Accounts: normalizedStrings(snapshot.Accounts), - Counts: cloneCounts(snapshot.Counts), + Services: mergedManifestStrings(old.Services, snapshot.Services, reuseEncrypted), + Accounts: mergedManifestStrings(old.Accounts, snapshot.Accounts, reuseEncrypted), + Counts: mergedManifestCounts(old.Counts, snapshot.Counts, updatedServices, reuseEncrypted), Shards: shards, } if manifest.Counts == nil { @@ -244,6 +252,50 @@ func writeSnapshot(ctx context.Context, cfg Config, snapshot Snapshot, old Manif return manifest, nil } +func snapshotServices(snapshot Snapshot) map[string]struct{} { + services := map[string]struct{}{} + for _, service := range snapshot.Services { + service = strings.TrimSpace(service) + if service != "" { + services[service] = struct{}{} + } + } + for _, shard := range snapshot.Shards { + service := strings.TrimSpace(shard.Service) + if service != "" { + services[service] = struct{}{} + } + } + return services +} + +func mergedManifestStrings(old, next []string, preserveOld bool) []string { + if !preserveOld { + return normalizedStrings(next) + } + return normalizedStrings(append(append([]string(nil), old...), next...)) +} + +func mergedManifestCounts(old, next map[string]int, updatedServices map[string]struct{}, preserveOld bool) map[string]int { + out := map[string]int{} + if preserveOld { + for key, value := range old { + service, _, _ := strings.Cut(key, ".") + if _, ok := updatedServices[service]; ok { + continue + } + out[key] = value + } + } + for key, value := range next { + out[key] = value + } + if len(out) == 0 { + return nil + } + return out +} + func writeShard(cfg Config, old Manifest, shard PlainShard, reuseEncrypted bool) (ShardEntry, error) { if strings.TrimSpace(shard.Service) == "" { return ShardEntry{}, fmt.Errorf("backup shard service is required") @@ -431,17 +483,6 @@ func sameStrings(a, b []string) bool { return true } -func cloneCounts(in map[string]int) map[string]int { - if len(in) == 0 { - return nil - } - out := make(map[string]int, len(in)) - for key, value := range in { - out[key] = value - } - return out -} - func removeStaleShards(repo string, shards []ShardEntry) error { keep := map[string]struct{}{} for _, shard := range shards { diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 999211e..508596f 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -287,6 +287,40 @@ func TestPushRemovesStaleEncryptedShards(t *testing.T) { } } +func TestPushPreservesUntouchedServices(t *testing.T) { + ctx, repo, config, _ := initTestBackup(t) + gmailPath := "data/gmail/acct/messages/2026/04/part-0001.jsonl.gz.age" + calendarPath := "data/calendar/acct/events/part-0001.jsonl.gz.age" + gmailShard := mustGmailMessageShard(t, gmailPath, []map[string]string{{"id": "m1", "raw": "body"}}) + calendarShard, err := NewJSONLShard("calendar", "events", "acct", calendarPath, []map[string]string{{"id": "event1"}}) + if err != nil { + t.Fatalf("NewJSONLShard calendar: %v", err) + } + pushSingleShard(t, ctx, config, gmailShard) + if _, err := PushSnapshot(ctx, Snapshot{ + Services: []string{"calendar"}, + Accounts: []string{"acct"}, + Counts: map[string]int{"calendar.events": 1}, + Shards: []PlainShard{calendarShard}, + }, Options{ConfigPath: config, Push: false}); err != nil { + t.Fatalf("PushSnapshot calendar: %v", err) + } + + manifest := readTestManifest(t, repo) + if _, ok := manifest.entry(gmailPath); !ok { + t.Fatal("gmail shard was removed by calendar-only push") + } + if _, ok := manifest.entry(calendarPath); !ok { + t.Fatal("calendar shard missing") + } + if manifest.Counts["gmail.messages"] != 1 || manifest.Counts["calendar.events"] != 1 { + t.Fatalf("counts = %+v, want preserved gmail and new calendar", manifest.Counts) + } + if _, err := os.Stat(filepath.Join(repo, filepath.FromSlash(gmailPath))); err != nil { + t.Fatalf("gmail shard file missing: %v", err) + } +} + func TestRejectsInvalidShardPaths(t *testing.T) { _, _, config, _ := initTestBackup(t) for _, rel := range []string{ diff --git a/internal/cmd/backup.go b/internal/cmd/backup.go index 7c49557..ca1a267 100644 --- a/internal/cmd/backup.go +++ b/internal/cmd/backup.go @@ -33,7 +33,13 @@ type BackupGmailCmd struct { Push BackupGmailPushCmd `cmd:"" name:"push" help:"Export Gmail into encrypted backup shards"` } -const backupServiceGmail = "gmail" +const ( + backupServiceCalendar = "calendar" + backupServiceContacts = "contacts" + backupServiceDrive = "drive" + backupServiceGmail = "gmail" + backupServiceTasks = "tasks" +) type backupFlags struct { Config string `name:"config" help:"Backup config path" default:""` @@ -103,17 +109,35 @@ type BackupPushCmd struct { Query string `name:"query" help:"Gmail query for bounded/test backups"` Max int64 `name:"max" aliases:"limit" help:"Max Gmail messages to export; 0 means all" default:"0"` IncludeSpamTrash bool `name:"include-spam-trash" help:"Include Gmail spam and trash" default:"true"` - ShardMaxRows int `name:"shard-max-rows" help:"Max Gmail messages per encrypted shard" default:"1000"` + ShardMaxRows int `name:"shard-max-rows" help:"Max rows per encrypted shard" default:"1000"` } func (c *BackupPushCmd) Run(ctx context.Context, flags *RootFlags) error { - services := splitCSV(c.Services) + services := expandBackupServices(splitCSV(c.Services)) if len(services) == 0 { return usage("at least one --services value is required") } var snapshots []backup.Snapshot for _, service := range services { switch strings.ToLower(strings.TrimSpace(service)) { + case backupServiceCalendar: + snapshot, err := buildCalendarBackupSnapshot(ctx, flags, c.ShardMaxRows) + if err != nil { + return err + } + snapshots = append(snapshots, snapshot) + case backupServiceContacts: + snapshot, err := buildContactsBackupSnapshot(ctx, flags, c.ShardMaxRows) + if err != nil { + return err + } + snapshots = append(snapshots, snapshot) + case backupServiceDrive: + snapshot, err := buildDriveBackupSnapshot(ctx, flags, c.ShardMaxRows) + if err != nil { + return err + } + snapshots = append(snapshots, snapshot) case backupServiceGmail: snapshot, err := buildGmailBackupSnapshot(ctx, flags, gmailBackupOptions{ Query: c.Query, @@ -125,8 +149,14 @@ func (c *BackupPushCmd) Run(ctx context.Context, flags *RootFlags) error { return err } snapshots = append(snapshots, snapshot) + case backupServiceTasks: + snapshot, err := buildTasksBackupSnapshot(ctx, flags, c.ShardMaxRows) + if err != nil { + return err + } + snapshots = append(snapshots, snapshot) default: - return fmt.Errorf("unsupported backup service %q (currently: gmail)", service) + return fmt.Errorf("unsupported backup service %q (supported: all, calendar, contacts, drive, gmail, tasks)", service) } } result, err := backup.PushSnapshot(ctx, mergeBackupSnapshots(snapshots...), c.options()) diff --git a/internal/cmd/backup_services.go b/internal/cmd/backup_services.go new file mode 100644 index 0000000..bcd3d5d --- /dev/null +++ b/internal/cmd/backup_services.go @@ -0,0 +1,465 @@ +package cmd + +import ( + "context" + "fmt" + "sort" + "strings" + + "google.golang.org/api/calendar/v3" + "google.golang.org/api/drive/v3" + gapi "google.golang.org/api/googleapi" + "google.golang.org/api/people/v1" + "google.golang.org/api/tasks/v1" + + "github.com/steipete/gogcli/internal/backup" +) + +type calendarBackupEvent struct { + CalendarID string `json:"calendarId"` + Event *calendar.Event `json:"event"` +} + +type contactsBackupPerson struct { + Source string `json:"source"` + Person *people.Person `json:"person"` +} + +type driveBackupFile struct { + File *drive.File `json:"file"` +} + +type tasksBackupTask struct { + TaskListID string `json:"taskListId"` + Task *tasks.Task `json:"task"` +} + +func expandBackupServices(services []string) []string { + var out []string + for _, service := range services { + if strings.EqualFold(strings.TrimSpace(service), "all") { + out = append(out, backupServiceCalendar, backupServiceContacts, backupServiceDrive, backupServiceGmail, backupServiceTasks) + continue + } + out = append(out, service) + } + return out +} + +func buildCalendarBackupSnapshot(ctx context.Context, flags *RootFlags, shardMaxRows int) (backup.Snapshot, error) { + account, err := requireAccount(flags) + if err != nil { + return backup.Snapshot{}, err + } + svc, err := newCalendarService(ctx, account) + if err != nil { + return backup.Snapshot{}, err + } + accountHash := backupAccountHash(account) + calendars, err := fetchBackupCalendars(ctx, svc) + if err != nil { + return backup.Snapshot{}, err + } + events, err := fetchBackupCalendarEvents(ctx, svc, calendars) + if err != nil { + return backup.Snapshot{}, err + } + shards := make([]backup.PlainShard, 0, 2) + calendarShard, err := backup.NewJSONLShard(backupServiceCalendar, "calendars", accountHash, fmt.Sprintf("data/calendar/%s/calendars.jsonl.gz.age", accountHash), calendars) + if err != nil { + return backup.Snapshot{}, err + } + shards = append(shards, calendarShard) + eventShards, err := buildBackupShards(backupServiceCalendar, "events", accountHash, fmt.Sprintf("data/calendar/%s/events", accountHash), events, shardMaxRows) + if err != nil { + return backup.Snapshot{}, err + } + shards = append(shards, eventShards...) + return backup.Snapshot{ + Services: []string{backupServiceCalendar}, + Accounts: []string{accountHash}, + Counts: map[string]int{ + "calendar.calendars": len(calendars), + "calendar.events": len(events), + }, + Shards: shards, + }, nil +} + +func buildContactsBackupSnapshot(ctx context.Context, flags *RootFlags, shardMaxRows int) (backup.Snapshot, error) { + account, err := requireAccount(flags) + if err != nil { + return backup.Snapshot{}, err + } + accountHash := backupAccountHash(account) + var peopleRows []contactsBackupPerson + contactsSvc, err := newPeopleContactsService(ctx, account) + if err != nil { + return backup.Snapshot{}, err + } + connections, err := fetchBackupConnections(ctx, contactsSvc) + if err != nil { + return backup.Snapshot{}, err + } + peopleRows = append(peopleRows, connections...) + otherSvc, err := newPeopleOtherContactsService(ctx, account) + if err != nil { + return backup.Snapshot{}, err + } + otherContacts, err := fetchBackupOtherContacts(ctx, otherSvc) + if err != nil { + return backup.Snapshot{}, err + } + peopleRows = append(peopleRows, otherContacts...) + shards, err := buildBackupShards(backupServiceContacts, "people", accountHash, fmt.Sprintf("data/contacts/%s/people", accountHash), peopleRows, shardMaxRows) + if err != nil { + return backup.Snapshot{}, err + } + return backup.Snapshot{ + Services: []string{backupServiceContacts}, + Accounts: []string{accountHash}, + Counts: map[string]int{ + "contacts.connections": len(connections), + "contacts.other": len(otherContacts), + "contacts.people": len(peopleRows), + }, + Shards: shards, + }, nil +} + +func buildDriveBackupSnapshot(ctx context.Context, flags *RootFlags, shardMaxRows int) (backup.Snapshot, error) { + account, err := requireAccount(flags) + if err != nil { + return backup.Snapshot{}, err + } + svc, err := newDriveService(ctx, account) + if err != nil { + return backup.Snapshot{}, err + } + accountHash := backupAccountHash(account) + drives, err := fetchBackupSharedDrives(ctx, svc) + if err != nil { + return backup.Snapshot{}, err + } + files, err := fetchBackupDriveFiles(ctx, svc) + if err != nil { + return backup.Snapshot{}, err + } + shards := make([]backup.PlainShard, 0, 2) + driveShard, err := backup.NewJSONLShard(backupServiceDrive, "drives", accountHash, fmt.Sprintf("data/drive/%s/drives.jsonl.gz.age", accountHash), drives) + if err != nil { + return backup.Snapshot{}, err + } + shards = append(shards, driveShard) + fileShards, err := buildBackupShards(backupServiceDrive, "files", accountHash, fmt.Sprintf("data/drive/%s/files", accountHash), files, shardMaxRows) + if err != nil { + return backup.Snapshot{}, err + } + shards = append(shards, fileShards...) + return backup.Snapshot{ + Services: []string{backupServiceDrive}, + Accounts: []string{accountHash}, + Counts: map[string]int{ + "drive.drives": len(drives), + "drive.files": len(files), + }, + Shards: shards, + }, nil +} + +func buildTasksBackupSnapshot(ctx context.Context, flags *RootFlags, shardMaxRows int) (backup.Snapshot, error) { + account, err := requireAccount(flags) + if err != nil { + return backup.Snapshot{}, err + } + svc, err := newTasksService(ctx, account) + if err != nil { + return backup.Snapshot{}, err + } + accountHash := backupAccountHash(account) + lists, err := fetchBackupTaskLists(ctx, svc) + if err != nil { + return backup.Snapshot{}, err + } + tasksRows, err := fetchBackupTasks(ctx, svc, lists) + if err != nil { + return backup.Snapshot{}, err + } + shards := make([]backup.PlainShard, 0, 2) + listShard, err := backup.NewJSONLShard(backupServiceTasks, "lists", accountHash, fmt.Sprintf("data/tasks/%s/lists.jsonl.gz.age", accountHash), lists) + if err != nil { + return backup.Snapshot{}, err + } + shards = append(shards, listShard) + taskShards, err := buildBackupShards(backupServiceTasks, "tasks", accountHash, fmt.Sprintf("data/tasks/%s/tasks", accountHash), tasksRows, shardMaxRows) + if err != nil { + return backup.Snapshot{}, err + } + shards = append(shards, taskShards...) + return backup.Snapshot{ + Services: []string{backupServiceTasks}, + Accounts: []string{accountHash}, + Counts: map[string]int{ + "tasks.lists": len(lists), + "tasks.tasks": len(tasksRows), + }, + Shards: shards, + }, nil +} + +func fetchBackupCalendars(ctx context.Context, svc *calendar.Service) ([]*calendar.CalendarListEntry, error) { + var out []*calendar.CalendarListEntry + pageToken := "" + for { + call := svc.CalendarList.List().MaxResults(250).Context(ctx) + if pageToken != "" { + call = call.PageToken(pageToken) + } + resp, err := call.Do() + if err != nil { + return nil, err + } + out = append(out, resp.Items...) + if resp.NextPageToken == "" { + break + } + pageToken = resp.NextPageToken + } + sort.Slice(out, func(i, j int) bool { return out[i].Id < out[j].Id }) + return out, nil +} + +func fetchBackupCalendarEvents(ctx context.Context, svc *calendar.Service, calendars []*calendar.CalendarListEntry) ([]calendarBackupEvent, error) { + var out []calendarBackupEvent + for _, cal := range calendars { + if cal == nil || strings.TrimSpace(cal.Id) == "" { + continue + } + pageToken := "" + for { + call := svc.Events.List(cal.Id). + MaxResults(2500). + ShowDeleted(true). + SingleEvents(false). + Context(ctx) + if pageToken != "" { + call = call.PageToken(pageToken) + } + resp, err := call.Do() + if err != nil { + return nil, fmt.Errorf("calendar %s events: %w", cal.Id, err) + } + for _, event := range resp.Items { + out = append(out, calendarBackupEvent{CalendarID: cal.Id, Event: event}) + } + if resp.NextPageToken == "" { + break + } + pageToken = resp.NextPageToken + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].CalendarID == out[j].CalendarID { + return out[i].Event.Id < out[j].Event.Id + } + return out[i].CalendarID < out[j].CalendarID + }) + return out, nil +} + +func fetchBackupConnections(ctx context.Context, svc *people.Service) ([]contactsBackupPerson, error) { + var out []contactsBackupPerson + pageToken := "" + for { + call := svc.People.Connections.List(peopleMeResource). + PersonFields(contactsGetReadMask). + PageSize(1000). + Context(ctx) + if pageToken != "" { + call = call.PageToken(pageToken) + } + resp, err := call.Do() + if err != nil { + return nil, err + } + for _, person := range resp.Connections { + out = append(out, contactsBackupPerson{Source: "connections", Person: person}) + } + if resp.NextPageToken == "" { + break + } + pageToken = resp.NextPageToken + } + return out, nil +} + +func fetchBackupOtherContacts(ctx context.Context, svc *people.Service) ([]contactsBackupPerson, error) { + const otherContactsBackupReadMask = "names,emailAddresses,phoneNumbers" + + var out []contactsBackupPerson + pageToken := "" + for { + call := svc.OtherContacts.List(). + ReadMask(otherContactsBackupReadMask). + PageSize(1000). + Context(ctx) + if pageToken != "" { + call = call.PageToken(pageToken) + } + resp, err := call.Do() + if err != nil { + return nil, err + } + for _, person := range resp.OtherContacts { + out = append(out, contactsBackupPerson{Source: "other", Person: person}) + } + if resp.NextPageToken == "" { + break + } + pageToken = resp.NextPageToken + } + return out, nil +} + +func fetchBackupSharedDrives(ctx context.Context, svc *drive.Service) ([]*drive.Drive, error) { + var out []*drive.Drive + pageToken := "" + for { + call := svc.Drives.List(). + PageSize(100). + Fields("nextPageToken, drives(id, name, createdTime, hidden, restrictions)"). + Context(ctx) + if pageToken != "" { + call = call.PageToken(pageToken) + } + resp, err := call.Do() + if err != nil { + return nil, err + } + out = append(out, resp.Drives...) + if resp.NextPageToken == "" { + break + } + pageToken = resp.NextPageToken + } + sort.Slice(out, func(i, j int) bool { return out[i].Id < out[j].Id }) + return out, nil +} + +func fetchBackupDriveFiles(ctx context.Context, svc *drive.Service) ([]driveBackupFile, error) { + var out []driveBackupFile + pageToken := "" + for { + call := svc.Files.List(). + Q("trashed = false"). + PageSize(1000). + OrderBy("modifiedTime desc"). + Fields(gapi.Field("nextPageToken, files(id, name, mimeType, size, createdTime, modifiedTime, viewedByMeTime, parents, owners, lastModifyingUser, webViewLink, webContentLink, description, starred, trashed, explicitlyTrashed, shared, ownedByMe, driveId, md5Checksum, sha1Checksum, sha256Checksum, originalFilename, fileExtension, exportLinks, appProperties, properties)")). + Context(ctx) + call = driveFilesListCallWithDriveSupport(call, true) + if pageToken != "" { + call = call.PageToken(pageToken) + } + resp, err := call.Do() + if err != nil { + return nil, err + } + for _, file := range resp.Files { + out = append(out, driveBackupFile{File: file}) + } + if resp.NextPageToken == "" { + break + } + pageToken = resp.NextPageToken + } + return out, nil +} + +func fetchBackupTaskLists(ctx context.Context, svc *tasks.Service) ([]*tasks.TaskList, error) { + var out []*tasks.TaskList + pageToken := "" + for { + call := svc.Tasklists.List().MaxResults(100).Context(ctx) + if pageToken != "" { + call = call.PageToken(pageToken) + } + resp, err := call.Do() + if err != nil { + return nil, err + } + out = append(out, resp.Items...) + if resp.NextPageToken == "" { + break + } + pageToken = resp.NextPageToken + } + sort.Slice(out, func(i, j int) bool { return out[i].Id < out[j].Id }) + return out, nil +} + +func fetchBackupTasks(ctx context.Context, svc *tasks.Service, lists []*tasks.TaskList) ([]tasksBackupTask, error) { + var out []tasksBackupTask + for _, list := range lists { + if list == nil || strings.TrimSpace(list.Id) == "" { + continue + } + pageToken := "" + for { + call := svc.Tasks.List(list.Id). + MaxResults(100). + ShowCompleted(true). + ShowDeleted(true). + ShowHidden(true). + ShowAssigned(true). + Context(ctx) + if pageToken != "" { + call = call.PageToken(pageToken) + } + resp, err := call.Do() + if err != nil { + return nil, fmt.Errorf("task list %s tasks: %w", list.Id, err) + } + for _, task := range resp.Items { + out = append(out, tasksBackupTask{TaskListID: list.Id, Task: task}) + } + if resp.NextPageToken == "" { + break + } + pageToken = resp.NextPageToken + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].TaskListID == out[j].TaskListID { + return out[i].Task.Id < out[j].Task.Id + } + return out[i].TaskListID < out[j].TaskListID + }) + return out, nil +} + +func buildBackupShards[T any](service, kind, accountHash, prefix string, rows []T, shardMaxRows int) ([]backup.PlainShard, error) { + if shardMaxRows <= 0 { + shardMaxRows = 1000 + } + if len(rows) == 0 { + shard, err := backup.NewJSONLShard(service, kind, accountHash, prefix+"/part-0001.jsonl.gz.age", rows) + if err != nil { + return nil, err + } + return []backup.PlainShard{shard}, nil + } + shards := make([]backup.PlainShard, 0, (len(rows)+shardMaxRows-1)/shardMaxRows) + for part, start := 1, 0; start < len(rows); part, start = part+1, start+shardMaxRows { + end := start + shardMaxRows + if end > len(rows) { + end = len(rows) + } + rel := fmt.Sprintf("%s/part-%04d.jsonl.gz.age", prefix, part) + shard, err := backup.NewJSONLShard(service, kind, accountHash, rel, rows[start:end]) + if err != nil { + return nil, err + } + shards = append(shards, shard) + } + return shards, nil +}