feat(backup): expand google backup coverage
This commit is contained in:
parent
8e48087457
commit
efc3df2ed8
@ -5,6 +5,7 @@
|
||||
### Added
|
||||
- Backup: add `gog backup` with age-encrypted Git shards, Gmail labels/raw message export, Calendar/Contacts/Tasks/Drive metadata adapters, manifest status, full decrypt-and-verify, shard `cat`, local plaintext export, docs, and security-focused regression coverage.
|
||||
- Backup: expand `gog backup push --services all` with Drive content export/download, Gmail settings, native Workspace Docs/Sheets/Slides/Form data, Apps Script projects, Chat, Classroom, best-effort optional service error shards, and plaintext Drive file export.
|
||||
- Backup: extend `--services all` with Drive permissions/comments/revisions, Calendar ACL/settings/colors, contact groups, Cloud Identity groups, Workspace Admin Directory users/groups/members, Keep notes, and local Gmail message caching for resumable full-mailbox fetches.
|
||||
|
||||
### Fixed
|
||||
- Gmail: auto-fill draft reply subjects from the original message when `gmail drafts create --reply-to-message-id` omits `--subject`. (#488) — thanks @jbowerbir.
|
||||
|
||||
11
README.md
11
README.md
@ -747,10 +747,13 @@ private age identity locally at `~/.gog/age.key`; GitHub only receives public
|
||||
The private `AGE-SECRET-KEY-...` value must stay local or in a password manager.
|
||||
|
||||
Supported backup services are `gmail`, `gmail-settings`, `calendar`,
|
||||
`contacts`, `tasks`, `drive`, `workspace`, `appscript`, `chat`, and
|
||||
`classroom`; `all` expands to those services. Drive now stores metadata plus
|
||||
exported Google-native file content by default. Non-Google binary Drive files
|
||||
are metadata-only unless `--drive-binary-contents` is set. Workspace inventories
|
||||
`contacts`, `tasks`, `drive`, `workspace`, `appscript`, `chat`, `classroom`,
|
||||
`groups`, `admin`, and `keep`; `all` expands to those services. Drive stores
|
||||
metadata, permissions, comments, revisions, and exported Google-native file
|
||||
content by default. Non-Google binary Drive files are metadata-only unless
|
||||
`--drive-binary-contents` is set. Gmail raw-message fetches use a local cache by
|
||||
default so interrupted full-mailbox backups can resume; use
|
||||
`--gmail-refresh-cache` to force a refetch. Workspace inventories
|
||||
Docs/Sheets/Slides and backs up Forms/responses discovered through Drive; add
|
||||
`--workspace-native` for full native Docs/Sheets/Slides API JSON.
|
||||
Optional Workspace-only services use `--best-effort` by default, recording
|
||||
|
||||
@ -71,18 +71,23 @@ remote.
|
||||
|
||||
Supported services:
|
||||
|
||||
- `gmail`: labels and raw MIME messages.
|
||||
- `gmail`: labels and raw MIME messages. Fetched raw messages are cached under
|
||||
the local user cache by default so interrupted full-mailbox runs can resume
|
||||
the expensive message download phase; use `--no-gmail-cache` or
|
||||
`--gmail-refresh-cache` to bypass it.
|
||||
- `gmail-settings`: filters, forwarding addresses, auto-forwarding, send-as
|
||||
aliases, vacation responder, delegate visibility, POP, IMAP, and language
|
||||
settings.
|
||||
- `calendar`: calendar list entries and all events, including deleted events.
|
||||
- `contacts`: People API contacts and other contacts.
|
||||
- `calendar`: calendar list entries, ACL rules, Calendar settings/colors, and
|
||||
all events, including deleted events.
|
||||
- `contacts`: People API contacts, other contacts, and contact groups.
|
||||
- `tasks`: task lists and tasks, including completed, deleted, hidden, and
|
||||
assigned tasks.
|
||||
- `drive`: shared drives, Drive file metadata, and downloaded/exported file
|
||||
content. Google Docs export as `.docx` and Markdown, Sheets as `.xlsx`,
|
||||
Slides as `.pptx` and PDF, Drawings as PNG and PDF, and binary files as
|
||||
metadata-only unless `--drive-binary-contents` is set.
|
||||
- `drive`: shared drives, Drive file metadata, permissions, comments, revision
|
||||
metadata, and downloaded/exported file content. Google Docs export as `.docx`
|
||||
and Markdown, Sheets as `.xlsx`, Slides as `.pptx` and PDF, Drawings as PNG
|
||||
and PDF, and binary files as metadata-only unless `--drive-binary-contents`
|
||||
is set.
|
||||
- `workspace`: Docs/Sheets/Slides inventory plus Forms and form responses
|
||||
discovered through Drive. Add `--workspace-native` to fetch full native
|
||||
Docs/Sheets/Slides API JSON.
|
||||
@ -92,13 +97,22 @@ Supported services:
|
||||
access.
|
||||
- `classroom`: courses, topics, announcements, coursework, materials, and
|
||||
submissions visible to the authenticated account.
|
||||
- `groups`: Cloud Identity groups the account belongs to, plus member lists
|
||||
when the API permits them.
|
||||
- `admin`: Workspace Admin Directory users, groups, and group members. This is
|
||||
Workspace-only and requires the existing Admin SDK/domain-wide delegation
|
||||
setup.
|
||||
- `keep`: Google Keep notes. This is Workspace-only and requires the existing
|
||||
Keep service-account setup.
|
||||
|
||||
`all` expands to every supported service. Pushing a subset updates that subset
|
||||
and preserves existing shards for services that were not selected, as long as
|
||||
the age recipients are unchanged.
|
||||
|
||||
`gog backup push` enables `--drive-contents` and `--best-effort` by default.
|
||||
Use `--no-drive-contents` for metadata-only Drive runs, or
|
||||
`gog backup push` enables `--drive-contents`, `--drive-collaboration`,
|
||||
`--gmail-cache`, and `--best-effort` by default. Use `--no-drive-contents` for
|
||||
metadata-only Drive runs, `--no-drive-collaboration` to skip per-file Drive
|
||||
permissions/comments/revisions, or
|
||||
`--drive-content-max-bytes <bytes>` to skip individual large Drive downloads.
|
||||
Drive content exports Google-native files by default; set
|
||||
`--drive-binary-contents` only when you intentionally want non-Google binary
|
||||
@ -106,7 +120,8 @@ file bytes in Git shards. Use `--workspace-native` only when you want the
|
||||
heavier native API JSON in addition to readable Drive exports;
|
||||
`--workspace-max-files` bounds that native fetch per file type for smoke tests.
|
||||
Best-effort optional services record encrypted `errors` shards and let the rest
|
||||
of the backup finish.
|
||||
of the backup finish. The Gmail cache is only a local acceleration/resume cache;
|
||||
encrypted backup shards remain the source of truth once a push completes.
|
||||
|
||||
## Files
|
||||
|
||||
@ -127,6 +142,9 @@ data/gmail/<account-hash>/messages/YYYY/MM/part-0001.jsonl.gz.age
|
||||
data/calendar/<account-hash>/...
|
||||
data/contacts/<account-hash>/...
|
||||
data/drive/<account-hash>/...
|
||||
data/groups/<account-hash>/...
|
||||
data/admin/<account-hash>/...
|
||||
data/keep/<account-hash>/...
|
||||
data/tasks/<account-hash>/...
|
||||
```
|
||||
|
||||
@ -238,6 +256,14 @@ Raw message payloads stay base64url encoded inside encrypted JSONL. This
|
||||
preserves the RFC 2822 message content while keeping the shard format text
|
||||
friendly.
|
||||
|
||||
By default, each fetched raw message is also cached locally under the OS user
|
||||
cache directory (`gogcli/backup/gmail/<account-hash>/raw-v1/`). The cache stores
|
||||
the same raw message row that will be encrypted into shards and is keyed by a
|
||||
SHA-256 of the Gmail message ID, so rerunning after an interruption can reuse
|
||||
already fetched messages. `--gmail-refresh-cache` forces a refetch. The cache is
|
||||
plaintext local data; clear it if the machine should not retain local mail
|
||||
copies outside the encrypted backup/export locations.
|
||||
|
||||
`--include-spam-trash` defaults to true. Use `--query` and `--max` for bounded
|
||||
test exports; omit them for a full mailbox scan.
|
||||
|
||||
@ -245,10 +271,11 @@ The Gmail settings adapter backs up account configuration through read-only
|
||||
settings endpoints. Some settings, such as delegates, can be forbidden for
|
||||
consumer accounts; those errors are kept inside the encrypted settings shard.
|
||||
|
||||
The Calendar adapter backs up calendar list entries and all events from each
|
||||
calendar. The Contacts adapter backs up contacts and other contacts. The Tasks
|
||||
adapter backs up task lists and tasks. The Drive adapter backs up shared drives,
|
||||
file metadata, and Google-native file exports by default. Content rows store
|
||||
The Calendar adapter backs up calendar list entries, ACLs, settings, colors,
|
||||
and all events from each calendar. The Contacts adapter backs up contacts, other
|
||||
contacts, and contact groups. The Tasks adapter backs up task lists and tasks.
|
||||
The Drive adapter backs up shared drives, file metadata, permissions, comments,
|
||||
revision metadata, and Google-native file exports by default. Content rows store
|
||||
base64 bytes inside encrypted JSONL so Git only sees ciphertext; plaintext
|
||||
export decodes them back into regular files. Non-Google binary Drive bytes are
|
||||
opt-in because personal Drives can easily contain tens of gigabytes.
|
||||
|
||||
@ -39,6 +39,9 @@ const (
|
||||
backupServiceDrive = "drive"
|
||||
backupServiceGmail = "gmail"
|
||||
backupServiceGmailSettings = "gmail-settings"
|
||||
backupServiceGroups = "groups"
|
||||
backupServiceAdmin = "admin"
|
||||
backupServiceKeep = "keep"
|
||||
backupServiceTasks = "tasks"
|
||||
backupServiceWorkspace = "workspace"
|
||||
)
|
||||
@ -115,8 +118,11 @@ type BackupPushCmd struct {
|
||||
DriveContents bool `name:"drive-contents" help:"Download/export Drive file contents into encrypted shards" default:"true" negatable:""`
|
||||
DriveBinaryContents bool `name:"drive-binary-contents" help:"Include non-Google Drive binary file bytes in encrypted shards"`
|
||||
DriveContentMaxBytes int64 `name:"drive-content-max-bytes" help:"Skip individual Drive content exports larger than this many bytes; 0 means unlimited" default:"0"`
|
||||
DriveCollaboration bool `name:"drive-collaboration" help:"Back up Drive permissions, comments, and revision metadata" default:"true" negatable:""`
|
||||
WorkspaceNative bool `name:"workspace-native" help:"Fetch full native Docs/Sheets/Slides API JSON in addition to Drive exports"`
|
||||
WorkspaceMaxFiles int `name:"workspace-max-files" help:"Max Docs/Sheets/Slides files per type for native Workspace metadata; 0 means all" default:"0"`
|
||||
GmailCache bool `name:"gmail-cache" help:"Cache fetched Gmail raw messages locally so interrupted full backups can resume" default:"true" negatable:""`
|
||||
GmailRefreshCache bool `name:"gmail-refresh-cache" help:"Refetch Gmail messages even when a local backup cache entry exists"`
|
||||
BestEffort bool `name:"best-effort" help:"Record optional service errors as backup rows and continue" default:"true" negatable:""`
|
||||
}
|
||||
|
||||
@ -170,6 +176,7 @@ func (c *BackupPushCmd) Run(ctx context.Context, flags *RootFlags) error {
|
||||
IncludeContents: c.DriveContents,
|
||||
IncludeBinary: c.DriveBinaryContents,
|
||||
MaxContentBytes: c.DriveContentMaxBytes,
|
||||
IncludeCollab: c.DriveCollaboration,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@ -181,6 +188,8 @@ func (c *BackupPushCmd) Run(ctx context.Context, flags *RootFlags) error {
|
||||
Max: c.Max,
|
||||
IncludeSpamTrash: c.IncludeSpamTrash,
|
||||
ShardMaxRows: c.ShardMaxRows,
|
||||
CacheMessages: c.GmailCache,
|
||||
RefreshCache: c.GmailRefreshCache,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@ -192,6 +201,30 @@ func (c *BackupPushCmd) Run(ctx context.Context, flags *RootFlags) error {
|
||||
return err
|
||||
}
|
||||
snapshots = append(snapshots, snapshot)
|
||||
case backupServiceGroups:
|
||||
snapshot, err := c.buildOptionalSnapshot(flags, backupServiceGroups, func() (backup.Snapshot, error) {
|
||||
return buildGroupsBackupSnapshot(ctx, flags, c.ShardMaxRows)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
snapshots = append(snapshots, snapshot)
|
||||
case backupServiceAdmin:
|
||||
snapshot, err := c.buildOptionalSnapshot(flags, backupServiceAdmin, func() (backup.Snapshot, error) {
|
||||
return buildAdminBackupSnapshot(ctx, flags, c.ShardMaxRows)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
snapshots = append(snapshots, snapshot)
|
||||
case backupServiceKeep:
|
||||
snapshot, err := c.buildOptionalSnapshot(flags, backupServiceKeep, func() (backup.Snapshot, error) {
|
||||
return buildKeepBackupSnapshot(ctx, flags, c.ShardMaxRows)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
snapshots = append(snapshots, snapshot)
|
||||
case backupServiceTasks:
|
||||
snapshot, err := buildTasksBackupSnapshot(ctx, flags, c.ShardMaxRows)
|
||||
if err != nil {
|
||||
@ -211,7 +244,7 @@ func (c *BackupPushCmd) Run(ctx context.Context, flags *RootFlags) error {
|
||||
}
|
||||
snapshots = append(snapshots, snapshot)
|
||||
default:
|
||||
return fmt.Errorf("unsupported backup service %q (supported: all, appscript, calendar, chat, classroom, contacts, drive, gmail, gmail-settings, tasks, workspace)", service)
|
||||
return fmt.Errorf("unsupported backup service %q (supported: all, admin, appscript, calendar, chat, classroom, contacts, drive, gmail, gmail-settings, groups, keep, tasks, workspace)", service)
|
||||
}
|
||||
}
|
||||
result, err := backup.PushSnapshot(ctx, mergeBackupSnapshots(snapshots...), c.options())
|
||||
@ -239,6 +272,8 @@ type BackupGmailPushCmd struct {
|
||||
Max int64 `name:"max" aliases:"limit" help:"Max Gmail messages to export; 0 means all" default:"0"`
|
||||
IncludeSpamTrash bool `name:"include-spam-trash" help:"Include spam and trash" default:"true"`
|
||||
ShardMaxRows int `name:"shard-max-rows" help:"Max messages per encrypted shard" default:"1000"`
|
||||
CacheMessages bool `name:"gmail-cache" help:"Cache fetched raw messages locally so interrupted full backups can resume" default:"true" negatable:""`
|
||||
RefreshCache bool `name:"gmail-refresh-cache" help:"Refetch messages even when a local backup cache entry exists"`
|
||||
}
|
||||
|
||||
func (c *BackupGmailPushCmd) Run(ctx context.Context, flags *RootFlags) error {
|
||||
@ -247,6 +282,8 @@ func (c *BackupGmailPushCmd) Run(ctx context.Context, flags *RootFlags) error {
|
||||
Max: c.Max,
|
||||
IncludeSpamTrash: c.IncludeSpamTrash,
|
||||
ShardMaxRows: c.ShardMaxRows,
|
||||
CacheMessages: c.CacheMessages,
|
||||
RefreshCache: c.RefreshCache,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
330
internal/cmd/backup_directory_keep.go
Normal file
330
internal/cmd/backup_directory_keep.go
Normal file
@ -0,0 +1,330 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
admin "google.golang.org/api/admin/directory/v1"
|
||||
"google.golang.org/api/cloudidentity/v1"
|
||||
keepapi "google.golang.org/api/keep/v1"
|
||||
|
||||
"github.com/steipete/gogcli/internal/backup"
|
||||
)
|
||||
|
||||
type groupsBackupMember struct {
|
||||
GroupEmail string `json:"groupEmail"`
|
||||
Member *cloudidentity.Membership `json:"member,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type adminBackupMember struct {
|
||||
GroupEmail string `json:"groupEmail"`
|
||||
Member *admin.Member `json:"member,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func buildGroupsBackupSnapshot(ctx context.Context, flags *RootFlags, shardMaxRows int) (backup.Snapshot, error) {
|
||||
account, err := requireAccount(flags)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
svc, err := newCloudIdentityService(ctx, account)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, wrapCloudIdentityError(err, account)
|
||||
}
|
||||
accountHash := backupAccountHash(account)
|
||||
groups, err := fetchBackupCloudIdentityGroups(ctx, svc, account)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
members := fetchBackupCloudIdentityGroupMembers(ctx, svc, groups)
|
||||
groupShards, err := buildBackupShards(backupServiceGroups, "groups", accountHash, fmt.Sprintf("data/groups/%s/groups", accountHash), groups, shardMaxRows)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
memberShards, err := buildBackupShards(backupServiceGroups, "members", accountHash, fmt.Sprintf("data/groups/%s/members", accountHash), members, shardMaxRows)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
return backup.Snapshot{
|
||||
Services: []string{backupServiceGroups},
|
||||
Accounts: []string{accountHash},
|
||||
Counts: map[string]int{
|
||||
"groups.groups": len(groups),
|
||||
"groups.members": len(members),
|
||||
},
|
||||
Shards: append(groupShards, memberShards...),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildAdminBackupSnapshot(ctx context.Context, flags *RootFlags, shardMaxRows int) (backup.Snapshot, error) {
|
||||
account, err := requireAdminAccount(flags)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
svc, err := newAdminDirectoryService(ctx, account)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, wrapAdminDirectoryError(err, account)
|
||||
}
|
||||
domain := domainFromAccount(account)
|
||||
accountHash := backupAccountHash(account)
|
||||
users, err := fetchBackupAdminUsers(ctx, svc, domain)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
groups, err := fetchBackupAdminGroups(ctx, svc, domain)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
members := fetchBackupAdminGroupMembers(ctx, svc, groups)
|
||||
userShards, err := buildBackupShards(backupServiceAdmin, "users", accountHash, fmt.Sprintf("data/admin/%s/users", accountHash), users, shardMaxRows)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
groupShards, err := buildBackupShards(backupServiceAdmin, "groups", accountHash, fmt.Sprintf("data/admin/%s/groups", accountHash), groups, shardMaxRows)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
memberShards, err := buildBackupShards(backupServiceAdmin, "members", accountHash, fmt.Sprintf("data/admin/%s/members", accountHash), members, shardMaxRows)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
shards := make([]backup.PlainShard, 0, len(userShards)+len(groupShards)+len(memberShards))
|
||||
shards = append(shards, userShards...)
|
||||
shards = append(shards, groupShards...)
|
||||
shards = append(shards, memberShards...)
|
||||
return backup.Snapshot{
|
||||
Services: []string{backupServiceAdmin},
|
||||
Accounts: []string{accountHash},
|
||||
Counts: map[string]int{
|
||||
"admin.users": len(users),
|
||||
"admin.groups": len(groups),
|
||||
"admin.members": len(members),
|
||||
},
|
||||
Shards: shards,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildKeepBackupSnapshot(ctx context.Context, flags *RootFlags, shardMaxRows int) (backup.Snapshot, error) {
|
||||
account, err := requireAccount(flags)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
svc, err := getKeepService(ctx, flags, &KeepCmd{})
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
accountHash := backupAccountHash(account)
|
||||
notes, err := fetchBackupKeepNotes(ctx, svc)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
shards, err := buildBackupShards(backupServiceKeep, "notes", accountHash, fmt.Sprintf("data/keep/%s/notes", accountHash), notes, shardMaxRows)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
return backup.Snapshot{
|
||||
Services: []string{backupServiceKeep},
|
||||
Accounts: []string{accountHash},
|
||||
Counts: map[string]int{"keep.notes": len(notes)},
|
||||
Shards: shards,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func fetchBackupCloudIdentityGroups(ctx context.Context, svc *cloudidentity.Service, account string) ([]*cloudidentity.GroupRelation, error) {
|
||||
var out []*cloudidentity.GroupRelation
|
||||
pageToken := ""
|
||||
for {
|
||||
call := svc.Groups.Memberships.SearchTransitiveGroups("groups/-").
|
||||
Query(searchTransitiveGroupsQuery(account)).
|
||||
PageSize(1000).
|
||||
Context(ctx)
|
||||
if pageToken != "" {
|
||||
call = call.PageToken(pageToken)
|
||||
}
|
||||
resp, err := call.Do()
|
||||
if err != nil {
|
||||
return nil, wrapCloudIdentityError(err, account)
|
||||
}
|
||||
out = append(out, resp.Memberships...)
|
||||
if resp.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return groupRelationEmail(out[i]) < groupRelationEmail(out[j]) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fetchBackupCloudIdentityGroupMembers(ctx context.Context, svc *cloudidentity.Service, groups []*cloudidentity.GroupRelation) []groupsBackupMember {
|
||||
var out []groupsBackupMember
|
||||
for _, group := range groups {
|
||||
groupEmail := groupRelationEmail(group)
|
||||
if groupEmail == "" {
|
||||
continue
|
||||
}
|
||||
groupName, err := lookupGroupByEmail(ctx, svc, groupEmail)
|
||||
if err != nil {
|
||||
out = append(out, groupsBackupMember{GroupEmail: groupEmail, Error: err.Error()})
|
||||
continue
|
||||
}
|
||||
pageToken := ""
|
||||
for {
|
||||
call := svc.Groups.Memberships.List(groupName).PageSize(1000).Context(ctx)
|
||||
if pageToken != "" {
|
||||
call = call.PageToken(pageToken)
|
||||
}
|
||||
resp, err := call.Do()
|
||||
if err != nil {
|
||||
out = append(out, groupsBackupMember{GroupEmail: groupEmail, Error: err.Error()})
|
||||
break
|
||||
}
|
||||
for _, member := range resp.Memberships {
|
||||
out = append(out, groupsBackupMember{GroupEmail: groupEmail, Member: member})
|
||||
}
|
||||
if resp.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].GroupEmail == out[j].GroupEmail {
|
||||
return cloudIdentityMemberSortKey(out[i].Member) < cloudIdentityMemberSortKey(out[j].Member)
|
||||
}
|
||||
return out[i].GroupEmail < out[j].GroupEmail
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func fetchBackupAdminUsers(ctx context.Context, svc *admin.Service, domain string) ([]*admin.User, error) {
|
||||
var out []*admin.User
|
||||
pageToken := ""
|
||||
for {
|
||||
call := svc.Users.List().Domain(domain).MaxResults(500).Context(ctx)
|
||||
if pageToken != "" {
|
||||
call = call.PageToken(pageToken)
|
||||
}
|
||||
resp, err := call.Do()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, resp.Users...)
|
||||
if resp.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].PrimaryEmail < out[j].PrimaryEmail })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fetchBackupAdminGroups(ctx context.Context, svc *admin.Service, domain string) ([]*admin.Group, error) {
|
||||
var out []*admin.Group
|
||||
pageToken := ""
|
||||
for {
|
||||
call := svc.Groups.List().Domain(domain).MaxResults(200).Context(ctx)
|
||||
if pageToken != "" {
|
||||
call = call.PageToken(pageToken)
|
||||
}
|
||||
resp, err := call.Do()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, resp.Groups...)
|
||||
if resp.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Email < out[j].Email })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fetchBackupAdminGroupMembers(ctx context.Context, svc *admin.Service, groups []*admin.Group) []adminBackupMember {
|
||||
var out []adminBackupMember
|
||||
for _, group := range groups {
|
||||
if group == nil || strings.TrimSpace(group.Email) == "" {
|
||||
continue
|
||||
}
|
||||
pageToken := ""
|
||||
for {
|
||||
call := svc.Members.List(group.Email).MaxResults(200).Context(ctx)
|
||||
if pageToken != "" {
|
||||
call = call.PageToken(pageToken)
|
||||
}
|
||||
resp, err := call.Do()
|
||||
if err != nil {
|
||||
out = append(out, adminBackupMember{GroupEmail: group.Email, Error: err.Error()})
|
||||
break
|
||||
}
|
||||
for _, member := range resp.Members {
|
||||
out = append(out, adminBackupMember{GroupEmail: group.Email, Member: member})
|
||||
}
|
||||
if resp.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].GroupEmail == out[j].GroupEmail {
|
||||
return adminMemberSortKey(out[i].Member) < adminMemberSortKey(out[j].Member)
|
||||
}
|
||||
return out[i].GroupEmail < out[j].GroupEmail
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func fetchBackupKeepNotes(ctx context.Context, svc *keepapi.Service) ([]*keepapi.Note, error) {
|
||||
var out []*keepapi.Note
|
||||
pageToken := ""
|
||||
for {
|
||||
call := svc.Notes.List().PageSize(1000).Context(ctx)
|
||||
if pageToken != "" {
|
||||
call = call.PageToken(pageToken)
|
||||
}
|
||||
resp, err := call.Do()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, resp.Notes...)
|
||||
if resp.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func domainFromAccount(account string) string {
|
||||
_, domain, ok := strings.Cut(strings.TrimSpace(account), "@")
|
||||
if !ok {
|
||||
return strings.TrimSpace(account)
|
||||
}
|
||||
return domain
|
||||
}
|
||||
|
||||
func groupRelationEmail(group *cloudidentity.GroupRelation) string {
|
||||
if group == nil || group.GroupKey == nil {
|
||||
return ""
|
||||
}
|
||||
return group.GroupKey.Id
|
||||
}
|
||||
|
||||
func cloudIdentityMemberSortKey(member *cloudidentity.Membership) string {
|
||||
if member == nil || member.PreferredMemberKey == nil {
|
||||
return ""
|
||||
}
|
||||
return member.PreferredMemberKey.Id
|
||||
}
|
||||
|
||||
func adminMemberSortKey(member *admin.Member) string {
|
||||
if member == nil {
|
||||
return ""
|
||||
}
|
||||
return member.Email
|
||||
}
|
||||
@ -20,6 +20,7 @@ type driveBackupOptions struct {
|
||||
IncludeContents bool
|
||||
IncludeBinary bool
|
||||
MaxContentBytes int64
|
||||
IncludeCollab bool
|
||||
}
|
||||
|
||||
func buildDriveBackupSnapshot(ctx context.Context, flags *RootFlags, opts driveBackupOptions) (backup.Snapshot, error) {
|
||||
@ -66,6 +67,27 @@ func buildDriveBackupSnapshot(ctx context.Context, flags *RootFlags, opts driveB
|
||||
counts[key] = value
|
||||
}
|
||||
}
|
||||
if opts.IncludeCollab {
|
||||
collab, collabCounts := fetchBackupDriveCollaboration(ctx, svc, files)
|
||||
permissionShards, shardErr := buildBackupShards(backupServiceDrive, "permissions", accountHash, fmt.Sprintf("data/drive/%s/permissions", accountHash), collab.Permissions, opts.ShardMaxRows)
|
||||
if shardErr != nil {
|
||||
return backup.Snapshot{}, shardErr
|
||||
}
|
||||
commentShards, shardErr := buildBackupShards(backupServiceDrive, "comments", accountHash, fmt.Sprintf("data/drive/%s/comments", accountHash), collab.Comments, opts.ShardMaxRows)
|
||||
if shardErr != nil {
|
||||
return backup.Snapshot{}, shardErr
|
||||
}
|
||||
revisionShards, shardErr := buildBackupShards(backupServiceDrive, "revisions", accountHash, fmt.Sprintf("data/drive/%s/revisions", accountHash), collab.Revisions, opts.ShardMaxRows)
|
||||
if shardErr != nil {
|
||||
return backup.Snapshot{}, shardErr
|
||||
}
|
||||
shards = append(shards, permissionShards...)
|
||||
shards = append(shards, commentShards...)
|
||||
shards = append(shards, revisionShards...)
|
||||
for key, value := range collabCounts {
|
||||
counts[key] = value
|
||||
}
|
||||
}
|
||||
return backup.Snapshot{
|
||||
Services: []string{backupServiceDrive},
|
||||
Accounts: []string{accountHash},
|
||||
|
||||
190
internal/cmd/backup_drive_collaboration.go
Normal file
190
internal/cmd/backup_drive_collaboration.go
Normal file
@ -0,0 +1,190 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"google.golang.org/api/drive/v3"
|
||||
gapi "google.golang.org/api/googleapi"
|
||||
)
|
||||
|
||||
type driveBackupCollaboration struct {
|
||||
Permissions []driveBackupPermission
|
||||
Comments []driveBackupComment
|
||||
Revisions []driveBackupRevision
|
||||
}
|
||||
|
||||
type driveBackupPermission struct {
|
||||
FileID string `json:"fileId"`
|
||||
Permission *drive.Permission `json:"permission,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type driveBackupComment struct {
|
||||
FileID string `json:"fileId"`
|
||||
Comment *drive.Comment `json:"comment,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type driveBackupRevision struct {
|
||||
FileID string `json:"fileId"`
|
||||
Revision *drive.Revision `json:"revision,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func fetchBackupDriveCollaboration(ctx context.Context, svc *drive.Service, files []driveBackupFile) (driveBackupCollaboration, map[string]int) {
|
||||
var out driveBackupCollaboration
|
||||
errors := 0
|
||||
for _, row := range files {
|
||||
if row.File == nil || strings.TrimSpace(row.File.Id) == "" {
|
||||
continue
|
||||
}
|
||||
fileID := row.File.Id
|
||||
permissions, err := fetchBackupDrivePermissions(ctx, svc, fileID)
|
||||
if err != nil {
|
||||
errors++
|
||||
out.Permissions = append(out.Permissions, driveBackupPermission{FileID: fileID, Error: err.Error()})
|
||||
} else {
|
||||
for _, permission := range permissions {
|
||||
out.Permissions = append(out.Permissions, driveBackupPermission{FileID: fileID, Permission: permission})
|
||||
}
|
||||
}
|
||||
comments, err := fetchBackupDriveComments(ctx, svc, fileID)
|
||||
if err != nil {
|
||||
errors++
|
||||
out.Comments = append(out.Comments, driveBackupComment{FileID: fileID, Error: err.Error()})
|
||||
} else {
|
||||
for _, comment := range comments {
|
||||
out.Comments = append(out.Comments, driveBackupComment{FileID: fileID, Comment: comment})
|
||||
}
|
||||
}
|
||||
revisions, err := fetchBackupDriveRevisions(ctx, svc, fileID)
|
||||
if err != nil {
|
||||
errors++
|
||||
out.Revisions = append(out.Revisions, driveBackupRevision{FileID: fileID, Error: err.Error()})
|
||||
} else {
|
||||
for _, revision := range revisions {
|
||||
out.Revisions = append(out.Revisions, driveBackupRevision{FileID: fileID, Revision: revision})
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(out.Permissions, func(i, j int) bool {
|
||||
if out.Permissions[i].FileID == out.Permissions[j].FileID {
|
||||
return drivePermissionSortKey(out.Permissions[i].Permission) < drivePermissionSortKey(out.Permissions[j].Permission)
|
||||
}
|
||||
return out.Permissions[i].FileID < out.Permissions[j].FileID
|
||||
})
|
||||
sort.Slice(out.Comments, func(i, j int) bool {
|
||||
if out.Comments[i].FileID == out.Comments[j].FileID {
|
||||
return driveCommentSortKey(out.Comments[i].Comment) < driveCommentSortKey(out.Comments[j].Comment)
|
||||
}
|
||||
return out.Comments[i].FileID < out.Comments[j].FileID
|
||||
})
|
||||
sort.Slice(out.Revisions, func(i, j int) bool {
|
||||
if out.Revisions[i].FileID == out.Revisions[j].FileID {
|
||||
return driveRevisionSortKey(out.Revisions[i].Revision) < driveRevisionSortKey(out.Revisions[j].Revision)
|
||||
}
|
||||
return out.Revisions[i].FileID < out.Revisions[j].FileID
|
||||
})
|
||||
return out, map[string]int{
|
||||
"drive.permissions": len(out.Permissions),
|
||||
"drive.comments": len(out.Comments),
|
||||
"drive.revisions": len(out.Revisions),
|
||||
"drive.collab.errors": errors,
|
||||
}
|
||||
}
|
||||
|
||||
func fetchBackupDrivePermissions(ctx context.Context, svc *drive.Service, fileID string) ([]*drive.Permission, error) {
|
||||
var out []*drive.Permission
|
||||
pageToken := ""
|
||||
for {
|
||||
call := svc.Permissions.List(fileID).
|
||||
PageSize(100).
|
||||
SupportsAllDrives(true).
|
||||
Fields(gapi.Field("nextPageToken, permissions(id,type,role,emailAddress,domain,displayName,allowFileDiscovery,deleted,expirationTime,inheritedPermissionsDisabled,pendingOwner,permissionDetails,photoLink,view)")).
|
||||
Context(ctx)
|
||||
if pageToken != "" {
|
||||
call = call.PageToken(pageToken)
|
||||
}
|
||||
resp, err := call.Do()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("drive file %s permissions: %w", fileID, err)
|
||||
}
|
||||
out = append(out, resp.Permissions...)
|
||||
if resp.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fetchBackupDriveComments(ctx context.Context, svc *drive.Service, fileID string) ([]*drive.Comment, error) {
|
||||
var out []*drive.Comment
|
||||
pageToken := ""
|
||||
for {
|
||||
comments, nextPageToken, err := listDriveComments(ctx, svc, fileID, driveCommentListOptions{
|
||||
includeResolved: true,
|
||||
includeQuoted: true,
|
||||
page: pageToken,
|
||||
max: 100,
|
||||
mode: driveCommentListModeExpanded,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("drive file %s comments: %w", fileID, err)
|
||||
}
|
||||
out = append(out, comments...)
|
||||
if nextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = nextPageToken
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fetchBackupDriveRevisions(ctx context.Context, svc *drive.Service, fileID string) ([]*drive.Revision, error) {
|
||||
var out []*drive.Revision
|
||||
pageToken := ""
|
||||
for {
|
||||
call := svc.Revisions.List(fileID).
|
||||
PageSize(200).
|
||||
Fields(gapi.Field("nextPageToken, revisions(id,mimeType,modifiedTime,keepForever,published,publishAuto,publishedOutsideDomain,publishedLink,lastModifyingUser,md5Checksum,size,originalFilename,exportLinks)")).
|
||||
Context(ctx)
|
||||
if pageToken != "" {
|
||||
call = call.PageToken(pageToken)
|
||||
}
|
||||
resp, err := call.Do()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("drive file %s revisions: %w", fileID, err)
|
||||
}
|
||||
out = append(out, resp.Revisions...)
|
||||
if resp.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func drivePermissionSortKey(permission *drive.Permission) string {
|
||||
if permission == nil {
|
||||
return ""
|
||||
}
|
||||
return permission.Id + "\x00" + permission.Type + "\x00" + permission.Role
|
||||
}
|
||||
|
||||
func driveCommentSortKey(comment *drive.Comment) string {
|
||||
if comment == nil {
|
||||
return ""
|
||||
}
|
||||
return comment.Id
|
||||
}
|
||||
|
||||
func driveRevisionSortKey(revision *drive.Revision) string {
|
||||
if revision == nil {
|
||||
return ""
|
||||
}
|
||||
return revision.ModifiedTime + "\x00" + revision.Id
|
||||
}
|
||||
@ -2,7 +2,12 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@ -18,6 +23,9 @@ type gmailBackupOptions struct {
|
||||
Max int64
|
||||
IncludeSpamTrash bool
|
||||
ShardMaxRows int
|
||||
AccountHash string
|
||||
CacheMessages bool
|
||||
RefreshCache bool
|
||||
}
|
||||
|
||||
type gmailBackupMessage struct {
|
||||
@ -55,6 +63,7 @@ func buildGmailBackupSnapshot(ctx context.Context, flags *RootFlags, opts gmailB
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
accountHash := backupAccountHash(account)
|
||||
opts.AccountHash = accountHash
|
||||
labels, err := fetchGmailBackupLabels(ctx, svc)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
@ -136,6 +145,17 @@ func fetchGmailBackupMessages(ctx context.Context, svc *gmail.Service, opts gmai
|
||||
results <- result{index: index, err: ctx.Err()}
|
||||
return
|
||||
}
|
||||
if opts.CacheMessages && !opts.RefreshCache {
|
||||
msg, ok, err := readGmailBackupMessageCache(opts.AccountHash, messageID)
|
||||
if err != nil {
|
||||
results <- result{index: index, err: err}
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
results <- result{index: index, msg: msg}
|
||||
return
|
||||
}
|
||||
}
|
||||
msg, err := svc.Users.Messages.Get("me", messageID).
|
||||
Format(gmailFormatRaw).
|
||||
Fields("id,threadId,historyId,internalDate,labelIds,sizeEstimate,raw").
|
||||
@ -149,7 +169,7 @@ func fetchGmailBackupMessages(ctx context.Context, svc *gmail.Service, opts gmai
|
||||
results <- result{index: index, err: fmt.Errorf("gmail message %s returned empty raw payload", messageID)}
|
||||
return
|
||||
}
|
||||
results <- result{index: index, msg: gmailBackupMessage{
|
||||
backupMsg := gmailBackupMessage{
|
||||
ID: msg.Id,
|
||||
ThreadID: msg.ThreadId,
|
||||
HistoryID: formatHistoryID(msg.HistoryId),
|
||||
@ -157,7 +177,14 @@ func fetchGmailBackupMessages(ctx context.Context, svc *gmail.Service, opts gmai
|
||||
LabelIDs: append([]string(nil), msg.LabelIds...),
|
||||
SizeEstimate: msg.SizeEstimate,
|
||||
Raw: msg.Raw,
|
||||
}}
|
||||
}
|
||||
if opts.CacheMessages {
|
||||
if err := writeGmailBackupMessageCache(opts.AccountHash, backupMsg); err != nil {
|
||||
results <- result{index: index, err: err}
|
||||
return
|
||||
}
|
||||
}
|
||||
results <- result{index: index, msg: backupMsg}
|
||||
}(i, id)
|
||||
}
|
||||
go func() {
|
||||
@ -178,6 +205,86 @@ func fetchGmailBackupMessages(ctx context.Context, svc *gmail.Service, opts gmai
|
||||
return ordered, nil
|
||||
}
|
||||
|
||||
func readGmailBackupMessageCache(accountHash, messageID string) (gmailBackupMessage, bool, error) {
|
||||
path, ok := gmailBackupMessageCachePath(accountHash, messageID)
|
||||
if !ok {
|
||||
return gmailBackupMessage{}, false, nil
|
||||
}
|
||||
data, err := os.ReadFile(path) //nolint:gosec // cache path is derived from the OS cache dir, account hash, and hashed message ID.
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return gmailBackupMessage{}, false, nil
|
||||
}
|
||||
return gmailBackupMessage{}, false, fmt.Errorf("read gmail backup cache %s: %w", path, err)
|
||||
}
|
||||
var msg gmailBackupMessage
|
||||
if err := json.Unmarshal(data, &msg); err != nil {
|
||||
return gmailBackupMessage{}, false, fmt.Errorf("decode gmail backup cache %s: %w", path, err)
|
||||
}
|
||||
if msg.ID != messageID {
|
||||
return gmailBackupMessage{}, false, fmt.Errorf("gmail backup cache %s has id %q, want %q", path, msg.ID, messageID)
|
||||
}
|
||||
if strings.TrimSpace(msg.Raw) == "" {
|
||||
return gmailBackupMessage{}, false, fmt.Errorf("gmail backup cache %s has empty raw payload", path)
|
||||
}
|
||||
return msg, true, nil
|
||||
}
|
||||
|
||||
func writeGmailBackupMessageCache(accountHash string, msg gmailBackupMessage) error {
|
||||
if strings.TrimSpace(msg.ID) == "" {
|
||||
return nil
|
||||
}
|
||||
path, ok := gmailBackupMessageCachePath(accountHash, msg.ID)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return fmt.Errorf("create gmail backup cache dir: %w", err)
|
||||
}
|
||||
data, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode gmail backup cache %s: %w", msg.ID, err)
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), ".message-*.json")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create gmail backup cache temp: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmpPath)
|
||||
return fmt.Errorf("write gmail backup cache temp: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
return fmt.Errorf("close gmail backup cache temp: %w", err)
|
||||
}
|
||||
if err := os.Chmod(tmpPath, 0o600); err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
return fmt.Errorf("chmod gmail backup cache temp: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmpPath, path); err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
return fmt.Errorf("replace gmail backup cache %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func gmailBackupMessageCachePath(accountHash, messageID string) (string, bool) {
|
||||
accountHash = strings.TrimSpace(accountHash)
|
||||
messageID = strings.TrimSpace(messageID)
|
||||
if accountHash == "" || messageID == "" {
|
||||
return "", false
|
||||
}
|
||||
dir, err := os.UserCacheDir()
|
||||
if err != nil || strings.TrimSpace(dir) == "" {
|
||||
return "", false
|
||||
}
|
||||
sum := sha256.Sum256([]byte(messageID))
|
||||
name := hex.EncodeToString(sum[:]) + ".json"
|
||||
return filepath.Join(dir, "gogcli", "backup", "gmail", accountHash, "raw-v1", name), true
|
||||
}
|
||||
|
||||
func listGmailBackupMessageIDs(ctx context.Context, svc *gmail.Service, opts gmailBackupOptions) ([]string, error) {
|
||||
var ids []string
|
||||
pageToken := ""
|
||||
|
||||
@ -19,6 +19,12 @@ type calendarBackupEvent struct {
|
||||
Event *calendar.Event `json:"event"`
|
||||
}
|
||||
|
||||
type calendarBackupACLRule struct {
|
||||
CalendarID string `json:"calendarId"`
|
||||
Rule *calendar.AclRule `json:"rule,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type contactsBackupPerson struct {
|
||||
Source string `json:"source"`
|
||||
Person *people.Person `json:"person"`
|
||||
@ -48,6 +54,9 @@ func expandBackupServices(services []string) []string {
|
||||
backupServiceDrive,
|
||||
backupServiceGmail,
|
||||
backupServiceGmailSettings,
|
||||
backupServiceGroups,
|
||||
backupServiceAdmin,
|
||||
backupServiceKeep,
|
||||
backupServiceTasks,
|
||||
backupServiceWorkspace,
|
||||
)
|
||||
@ -94,6 +103,15 @@ func buildCalendarBackupSnapshot(ctx context.Context, flags *RootFlags, shardMax
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
aclRules := fetchBackupCalendarACLRules(ctx, svc, calendars)
|
||||
settings, err := fetchBackupCalendarSettings(ctx, svc)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
colors, err := svc.Colors.Get().Context(ctx).Do()
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
shards := make([]backup.PlainShard, 0, 2)
|
||||
calendarShard, err := backup.NewJSONLShard(backupServiceCalendar, "calendars", accountHash, fmt.Sprintf("data/calendar/%s/calendars.jsonl.gz.age", accountHash), calendars)
|
||||
if err != nil {
|
||||
@ -105,12 +123,30 @@ func buildCalendarBackupSnapshot(ctx context.Context, flags *RootFlags, shardMax
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
shards = append(shards, eventShards...)
|
||||
aclShards, err := buildBackupShards(backupServiceCalendar, "acl", accountHash, fmt.Sprintf("data/calendar/%s/acl", accountHash), aclRules, shardMaxRows)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
shards = append(shards, aclShards...)
|
||||
settingsShard, err := backup.NewJSONLShard(backupServiceCalendar, "settings", accountHash, fmt.Sprintf("data/calendar/%s/settings.jsonl.gz.age", accountHash), settings)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
shards = append(shards, settingsShard)
|
||||
colorsShard, err := backup.NewJSONLShard(backupServiceCalendar, "colors", accountHash, fmt.Sprintf("data/calendar/%s/colors.jsonl.gz.age", accountHash), []*calendar.Colors{colors})
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
shards = append(shards, colorsShard)
|
||||
return backup.Snapshot{
|
||||
Services: []string{backupServiceCalendar},
|
||||
Accounts: []string{accountHash},
|
||||
Counts: map[string]int{
|
||||
"calendar.calendars": len(calendars),
|
||||
"calendar.events": len(events),
|
||||
"calendar.acl": len(aclRules),
|
||||
"calendar.settings": len(settings),
|
||||
"calendar.colors": 1,
|
||||
},
|
||||
Shards: shards,
|
||||
}, nil
|
||||
@ -141,15 +177,25 @@ func buildContactsBackupSnapshot(ctx context.Context, flags *RootFlags, shardMax
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
peopleRows = append(peopleRows, otherContacts...)
|
||||
groups, err := fetchBackupContactGroups(ctx, contactsSvc)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
shards, err := buildBackupShards(backupServiceContacts, "people", accountHash, fmt.Sprintf("data/contacts/%s/people", accountHash), peopleRows, shardMaxRows)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
groupShard, err := backup.NewJSONLShard(backupServiceContacts, "groups", accountHash, fmt.Sprintf("data/contacts/%s/groups.jsonl.gz.age", accountHash), groups)
|
||||
if err != nil {
|
||||
return backup.Snapshot{}, err
|
||||
}
|
||||
shards = append(shards, groupShard)
|
||||
return backup.Snapshot{
|
||||
Services: []string{backupServiceContacts},
|
||||
Accounts: []string{accountHash},
|
||||
Counts: map[string]int{
|
||||
"contacts.connections": len(connections),
|
||||
"contacts.groups": len(groups),
|
||||
"contacts.other": len(otherContacts),
|
||||
"contacts.people": len(peopleRows),
|
||||
},
|
||||
@ -157,6 +203,74 @@ func buildContactsBackupSnapshot(ctx context.Context, flags *RootFlags, shardMax
|
||||
}, nil
|
||||
}
|
||||
|
||||
func fetchBackupCalendarACLRules(ctx context.Context, svc *calendar.Service, calendars []*calendar.CalendarListEntry) []calendarBackupACLRule {
|
||||
var out []calendarBackupACLRule
|
||||
for _, cal := range calendars {
|
||||
if cal == nil || strings.TrimSpace(cal.Id) == "" {
|
||||
continue
|
||||
}
|
||||
pageToken := ""
|
||||
for {
|
||||
call := svc.Acl.List(cal.Id).MaxResults(250).Context(ctx)
|
||||
if pageToken != "" {
|
||||
call = call.PageToken(pageToken)
|
||||
}
|
||||
resp, err := call.Do()
|
||||
if err != nil {
|
||||
out = append(out, calendarBackupACLRule{CalendarID: cal.Id, Error: err.Error()})
|
||||
break
|
||||
}
|
||||
for _, rule := range resp.Items {
|
||||
out = append(out, calendarBackupACLRule{CalendarID: cal.Id, Rule: rule})
|
||||
}
|
||||
if resp.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].CalendarID == out[j].CalendarID {
|
||||
return calendarACLRuleSortKey(out[i].Rule) < calendarACLRuleSortKey(out[j].Rule)
|
||||
}
|
||||
return out[i].CalendarID < out[j].CalendarID
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func fetchBackupCalendarSettings(ctx context.Context, svc *calendar.Service) ([]*calendar.Setting, error) {
|
||||
var out []*calendar.Setting
|
||||
pageToken := ""
|
||||
for {
|
||||
call := svc.Settings.List().MaxResults(250).Context(ctx)
|
||||
if pageToken != "" {
|
||||
call = call.PageToken(pageToken)
|
||||
}
|
||||
resp, err := call.Do()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, resp.Items...)
|
||||
if resp.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Id < out[j].Id })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func calendarACLRuleSortKey(rule *calendar.AclRule) string {
|
||||
if rule == nil {
|
||||
return ""
|
||||
}
|
||||
scope := ""
|
||||
if rule.Scope != nil {
|
||||
scope = rule.Scope.Type + "\x00" + rule.Scope.Value
|
||||
}
|
||||
return scope + "\x00" + rule.Role + "\x00" + rule.Id
|
||||
}
|
||||
|
||||
func buildTasksBackupSnapshot(ctx context.Context, flags *RootFlags, shardMaxRows int) (backup.Snapshot, error) {
|
||||
account, err := requireAccount(flags)
|
||||
if err != nil {
|
||||
@ -311,6 +425,31 @@ func fetchBackupOtherContacts(ctx context.Context, svc *people.Service) ([]conta
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fetchBackupContactGroups(ctx context.Context, svc *people.Service) ([]*people.ContactGroup, error) {
|
||||
var out []*people.ContactGroup
|
||||
pageToken := ""
|
||||
for {
|
||||
call := svc.ContactGroups.List().
|
||||
PageSize(1000).
|
||||
GroupFields("clientData,formattedName,groupType,memberCount,metadata,name,resourceName").
|
||||
Context(ctx)
|
||||
if pageToken != "" {
|
||||
call = call.PageToken(pageToken)
|
||||
}
|
||||
resp, err := call.Do()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, resp.ContactGroups...)
|
||||
if resp.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.NextPageToken
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ResourceName < out[j].ResourceName })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fetchBackupTaskLists(ctx context.Context, svc *tasks.Service) ([]*tasks.TaskList, error) {
|
||||
var out []*tasks.TaskList
|
||||
pageToken := ""
|
||||
|
||||
@ -2,6 +2,9 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@ -9,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/option"
|
||||
|
||||
"github.com/steipete/gogcli/internal/backup"
|
||||
)
|
||||
@ -104,6 +108,9 @@ func TestExpandBackupServicesAllIncludesWorkspaceAdapters(t *testing.T) {
|
||||
"drive",
|
||||
"gmail",
|
||||
"gmail-settings",
|
||||
"groups",
|
||||
"admin",
|
||||
"keep",
|
||||
"tasks",
|
||||
"workspace",
|
||||
} {
|
||||
@ -113,6 +120,119 @@ func TestExpandBackupServicesAllIncludesWorkspaceAdapters(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGmailBackupMessageCacheRoundTrips(t *testing.T) {
|
||||
t.Setenv("HOME", t.TempDir())
|
||||
message := gmailBackupMessage{
|
||||
ID: "msg-one",
|
||||
ThreadID: "thread-one",
|
||||
InternalDate: mustUnixMilli(t, "2026-04-02T10:00:00Z"),
|
||||
LabelIDs: []string{"INBOX"},
|
||||
Raw: base64.RawURLEncoding.EncodeToString([]byte("Subject: Cached\r\n\r\nBody")),
|
||||
}
|
||||
if err := writeGmailBackupMessageCache("accthash", message); err != nil {
|
||||
t.Fatalf("writeGmailBackupMessageCache: %v", err)
|
||||
}
|
||||
got, ok, err := readGmailBackupMessageCache("accthash", "msg-one")
|
||||
if err != nil {
|
||||
t.Fatalf("readGmailBackupMessageCache: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected cache hit")
|
||||
}
|
||||
if got.ID != message.ID || got.ThreadID != message.ThreadID || got.Raw != message.Raw {
|
||||
t.Fatalf("cache round trip mismatch: %#v", got)
|
||||
}
|
||||
path, ok := gmailBackupMessageCachePath("accthash", "msg-one")
|
||||
if !ok {
|
||||
t.Fatal("expected cache path")
|
||||
}
|
||||
if strings.Contains(path, "msg-one") {
|
||||
t.Fatalf("cache path should hash message IDs, got %q", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGmailBackupMessageCacheRejectsWrongID(t *testing.T) {
|
||||
t.Setenv("HOME", t.TempDir())
|
||||
message := gmailBackupMessage{ID: "msg-one", Raw: "raw"}
|
||||
if err := writeGmailBackupMessageCache("accthash", message); err != nil {
|
||||
t.Fatalf("writeGmailBackupMessageCache: %v", err)
|
||||
}
|
||||
path, ok := gmailBackupMessageCachePath("accthash", "msg-one")
|
||||
if !ok {
|
||||
t.Fatal("expected cache path")
|
||||
}
|
||||
data, err := json.Marshal(gmailBackupMessage{ID: "other", Raw: "raw"})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
if _, _, err := readGmailBackupMessageCache("accthash", "msg-one"); err == nil {
|
||||
t.Fatal("expected wrong cache ID to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchBackupDriveCollaborationCollectsMetadataAndErrors(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch {
|
||||
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/files/file1/permissions"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"permissions": []map[string]any{{"id": "perm1", "type": "user", "role": "reader", "emailAddress": "a@example.com"}},
|
||||
})
|
||||
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/files/file1/comments"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"comments": []map[string]any{{"id": "comment1", "content": "hello", "resolved": false}},
|
||||
})
|
||||
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/files/file1/revisions"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"revisions": []map[string]any{{"id": "rev1", "modifiedTime": "2026-04-02T10:00:00Z"}},
|
||||
})
|
||||
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/files/file2/permissions"):
|
||||
http.Error(w, `{"error":{"message":"denied"}}`, http.StatusForbidden)
|
||||
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/files/file2/comments"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{})
|
||||
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/files/file2/revisions"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
svc, err := drive.NewService(t.Context(),
|
||||
option.WithoutAuthentication(),
|
||||
option.WithHTTPClient(srv.Client()),
|
||||
option.WithEndpoint(srv.URL+"/"),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService: %v", err)
|
||||
}
|
||||
got, counts := fetchBackupDriveCollaboration(t.Context(), svc, []driveBackupFile{
|
||||
{File: &drive.File{Id: "file1"}},
|
||||
{File: &drive.File{Id: "file2"}},
|
||||
})
|
||||
if counts["drive.permissions"] != 2 || counts["drive.comments"] != 1 || counts["drive.revisions"] != 1 || counts["drive.collab.errors"] != 1 {
|
||||
t.Fatalf("unexpected counts: %#v", counts)
|
||||
}
|
||||
if got.Permissions[0].FileID != "file1" || got.Permissions[0].Permission.Id != "perm1" {
|
||||
t.Fatalf("unexpected permission row: %#v", got.Permissions[0])
|
||||
}
|
||||
if got.Permissions[1].FileID != "file2" || got.Permissions[1].Error == "" {
|
||||
t.Fatalf("expected file2 permission error row: %#v", got.Permissions[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDomainFromAccount(t *testing.T) {
|
||||
if got := domainFromAccount("Admin@Example.COM"); got != "Example.COM" {
|
||||
t.Fatalf("domainFromAccount = %q", got)
|
||||
}
|
||||
if got := domainFromAccount("example.com"); got != "example.com" {
|
||||
t.Fatalf("domainFromAccount without @ = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriveBackupContentPlansPreferReadableWorkspaceFormats(t *testing.T) {
|
||||
docPlans := driveBackupContentPlans(&drive.File{Id: "doc1", Name: "Spec", MimeType: driveMimeGoogleDoc}, false)
|
||||
if len(docPlans) != 2 || docPlans[0].Extension != ".docx" || docPlans[1].Extension != ".md" {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user