diff --git a/Pods b/Pods index 02d9e07b52..5564eb7e18 160000 --- a/Pods +++ b/Pods @@ -1 +1 @@ -Subproject commit 02d9e07b52a3c3a43cc29a90c586b61fc30cd032 +Subproject commit 5564eb7e1870233872738ab652793883d1dc1c3d diff --git a/Signal.xcodeproj/project.pbxproj b/Signal.xcodeproj/project.pbxproj index 4b116e5de0..18f0850457 100644 --- a/Signal.xcodeproj/project.pbxproj +++ b/Signal.xcodeproj/project.pbxproj @@ -40,6 +40,7 @@ 340FC8C7204DE64D007AEB0F /* OWSBackupAPI.swift in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8C6204DE64D007AEB0F /* OWSBackupAPI.swift */; }; 340FC8CA20517B84007AEB0F /* OWSBackupImportJob.m in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8C820517B84007AEB0F /* OWSBackupImportJob.m */; }; 340FC8CD20518C77007AEB0F /* OWSBackupJob.m in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8CC20518C76007AEB0F /* OWSBackupJob.m */; }; + 340FC8D0205BF2FA007AEB0F /* OWSBackupIO.m in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8CE205BF2FA007AEB0F /* OWSBackupIO.m */; }; 341F2C0F1F2B8AE700D07D6B /* DebugUIMisc.m in Sources */ = {isa = PBXBuildFile; fileRef = 341F2C0E1F2B8AE700D07D6B /* DebugUIMisc.m */; }; 3430FE181F7751D4000EC51B /* GiphyAPI.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3430FE171F7751D4000EC51B /* GiphyAPI.swift */; }; 34330A5A1E7875FB00DF2FB9 /* fontawesome-webfont.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 34330A591E7875FB00DF2FB9 /* fontawesome-webfont.ttf */; }; @@ -588,6 +589,8 @@ 340FC8C920517B84007AEB0F /* OWSBackupImportJob.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSBackupImportJob.h; sourceTree = ""; }; 340FC8CB20518C76007AEB0F /* OWSBackupJob.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSBackupJob.h; sourceTree = ""; }; 340FC8CC20518C76007AEB0F /* OWSBackupJob.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSBackupJob.m; sourceTree = ""; }; + 340FC8CE205BF2FA007AEB0F /* OWSBackupIO.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSBackupIO.m; sourceTree = ""; }; + 340FC8CF205BF2FA007AEB0F /* OWSBackupIO.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSBackupIO.h; sourceTree = ""; }; 341458471FBE11C4005ABCF9 /* fa */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = fa; path = translations/fa.lproj/Localizable.strings; sourceTree = ""; }; 341F2C0D1F2B8AE700D07D6B /* DebugUIMisc.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DebugUIMisc.h; sourceTree = ""; }; 341F2C0E1F2B8AE700D07D6B /* DebugUIMisc.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = DebugUIMisc.m; sourceTree = ""; }; @@ -1955,6 +1958,8 @@ 34A9105E1FFEB113000C4745 /* OWSBackup.h */, 34A9105F1FFEB114000C4745 /* OWSBackup.m */, 340FC8C6204DE64D007AEB0F /* OWSBackupAPI.swift */, + 340FC8CF205BF2FA007AEB0F /* OWSBackupIO.h */, + 340FC8CE205BF2FA007AEB0F /* OWSBackupIO.m */, 340FC8BE204DB7D1007AEB0F /* OWSBackupExportJob.h */, 340FC8BF204DB7D2007AEB0F /* OWSBackupExportJob.m */, 340FC8C920517B84007AEB0F /* OWSBackupImportJob.h */, @@ -3172,6 +3177,7 @@ 34D1F0B71F87F8850066283D /* OWSGenericAttachmentView.m in Sources */, 34B3F8801E8DF1700035BE1A /* InviteFlow.swift in Sources */, 457C87B82032645C008D52D6 /* DebugUINotifications.swift in Sources */, + 340FC8D0205BF2FA007AEB0F /* OWSBackupIO.m in Sources */, 458E38371D668EBF0094BD24 /* OWSDeviceProvisioningURLParser.m in Sources */, 4517642B1DE939FD00EDB8B9 /* ContactCell.swift in Sources */, 340FC8AB204DAC8D007AEB0F /* DomainFrontingCountryViewController.m in Sources */, diff --git a/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m b/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m index 4652b6294a..babc5d5adb 100644 --- a/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m +++ b/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m @@ -72,18 +72,28 @@ #ifdef DEBUG __block NSUInteger threadCount; __block NSUInteger messageCount; + __block NSUInteger attachmentCount; [OWSPrimaryStorage.dbReadConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) { - threadCount = [[transaction ext:TSThreadDatabaseViewExtensionName] numberOfItemsInAllGroups]; - messageCount = [[transaction ext:TSMessageDatabaseViewExtensionName] numberOfItemsInAllGroups]; + threadCount = [transaction numberOfKeysInCollection:[TSThread collection]]; + messageCount = [transaction numberOfKeysInCollection:[TSInteraction collection]]; + attachmentCount = [transaction numberOfKeysInCollection:[TSAttachment collection]]; }]; - unsigned long long databaseFileSize = [OWSPrimaryStorage.sharedManager databaseFileSize]; OWSTableSection *debugSection = [OWSTableSection new]; debugSection.headerTitle = @"Debug"; [debugSection addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Threads: %zd", threadCount]]]; [debugSection addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Messages: %zd", messageCount]]]; [debugSection - addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database size: %llu", databaseFileSize]]]; + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Attachments: %zd", attachmentCount]]]; + [debugSection + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database size: %llu", + [OWSPrimaryStorage.sharedManager databaseFileSize]]]]; + [debugSection + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database WAL size: %llu", + [OWSPrimaryStorage.sharedManager databaseWALFileSize]]]]; + [debugSection + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database SHM size: %llu", + [OWSPrimaryStorage.sharedManager databaseSHMFileSize]]]]; [contents addSection:debugSection]; OWSPreferences *preferences = [Environment preferences]; diff --git a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m index ee9d39b33a..8f46dcbaff 100644 --- a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m +++ b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m @@ -40,6 +40,10 @@ NS_ASSUME_NONNULL_BEGIN actionBlock:^{ [DebugUIBackup tryToImportBackup]; }]]; + [items addObject:[OWSTableItem itemWithTitle:@"Log Database Size Stats" + actionBlock:^{ + [DebugUIBackup logDatabaseSizeStats]; + }]]; return [OWSTableSection sectionWithTitle:self.name items:items]; } @@ -95,17 +99,57 @@ NS_ASSUME_NONNULL_BEGIN message:@"This will delete all of your database contents." preferredStyle:UIAlertControllerStyleAlert]; - [controller addAction:[UIAlertAction - actionWithTitle:@"Restore" - style:UIAlertActionStyleDefault - handler:^(UIAlertAction *_Nonnull action) { - [OWSBackup.sharedManager tryToImportBackup]; - }]]; + [controller addAction:[UIAlertAction actionWithTitle:@"Restore" + style:UIAlertActionStyleDefault + handler:^(UIAlertAction *_Nonnull action) { + [OWSBackup.sharedManager tryToImportBackup]; + }]]; [controller addAction:[OWSAlerts cancelAction]]; UIViewController *fromViewController = [[UIApplication sharedApplication] frontmostViewController]; [fromViewController presentViewController:controller animated:YES completion:nil]; } ++ (void)logDatabaseSizeStats +{ + DDLogInfo(@"%@ logDatabaseSizeStats.", self.logTag); + + __block unsigned long long interactionCount = 0; + __block unsigned long long interactionSizeTotal = 0; + __block unsigned long long attachmentCount = 0; + __block unsigned long long attachmentSizeTotal = 0; + [[OWSPrimaryStorage.sharedManager newDatabaseConnection] readWithBlock:^(YapDatabaseReadTransaction *transaction) { + [transaction enumerateKeysAndObjectsInCollection:[TSInteraction collection] + usingBlock:^(NSString *key, id object, BOOL *stop) { + TSInteraction *interaction = object; + interactionCount++; + NSData *_Nullable data = + [NSKeyedArchiver archivedDataWithRootObject:interaction]; + OWSAssert(data); + interactionSizeTotal += data.length; + }]; + [transaction enumerateKeysAndObjectsInCollection:[TSAttachment collection] + usingBlock:^(NSString *key, id object, BOOL *stop) { + TSAttachment *attachment = object; + attachmentCount++; + NSData *_Nullable data = + [NSKeyedArchiver archivedDataWithRootObject:attachment]; + OWSAssert(data); + attachmentSizeTotal += data.length; + }]; + }]; + + DDLogInfo(@"%@ interactionCount: %llu", self.logTag, interactionCount); + DDLogInfo(@"%@ interactionSizeTotal: %llu", self.logTag, interactionSizeTotal); + if (interactionCount > 0) { + DDLogInfo(@"%@ interaction average size: %f", self.logTag, interactionSizeTotal / (double)interactionCount); + } + DDLogInfo(@"%@ attachmentCount: %llu", self.logTag, attachmentCount); + DDLogInfo(@"%@ attachmentSizeTotal: %llu", self.logTag, attachmentSizeTotal); + if (attachmentCount > 0) { + DDLogInfo(@"%@ attachment average size: %f", self.logTag, attachmentSizeTotal / (double)attachmentCount); + } +} + @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackup.m b/Signal/src/util/OWSBackup.m index d05368211b..504e1abd37 100644 --- a/Signal/src/util/OWSBackup.m +++ b/Signal/src/util/OWSBackup.m @@ -13,7 +13,6 @@ NSString *const NSNotificationNameBackupStateDidChange = @"NSNotificationNameBac NSString *const OWSPrimaryStorage_OWSBackupCollection = @"OWSPrimaryStorage_OWSBackupCollection"; NSString *const OWSBackup_IsBackupEnabledKey = @"OWSBackup_IsBackupEnabledKey"; -NSString *const OWSBackup_BackupKeyKey = @"OWSBackup_BackupKeyKey"; NSString *const OWSBackup_LastExportSuccessDateKey = @"OWSBackup_LastExportSuccessDateKey"; NSString *const OWSBackup_LastExportFailureDateKey = @"OWSBackup_LastExportFailureDateKey"; @@ -107,31 +106,6 @@ NS_ASSUME_NONNULL_BEGIN }); } -- (void)setBackupPrivateKey:(NSData *)value -{ - OWSAssert(value); - - // TODO: Use actual key. - [self.dbConnection setObject:value - forKey:OWSBackup_BackupKeyKey - inCollection:OWSPrimaryStorage_OWSBackupCollection]; -} - -- (nullable NSData *)backupPrivateKey -{ - NSData *_Nullable result = - [self.dbConnection objectForKey:OWSBackup_BackupKeyKey inCollection:OWSPrimaryStorage_OWSBackupCollection]; - if (!result) { - // TODO: Use actual key. - const NSUInteger kBackupPrivateKeyLength = 32; - result = [Randomness generateRandomBytes:kBackupPrivateKeyLength]; - [self setBackupPrivateKey:result]; - } - OWSAssert(result); - OWSAssert([result isKindOfClass:[NSData class]]); - return result; -} - #pragma mark - Backup Export - (void)tryToExportBackup @@ -383,9 +357,10 @@ NS_ASSUME_NONNULL_BEGIN #pragma mark - OWSBackupJobDelegate // We use a delegate method to avoid storing this key in memory. -- (nullable NSData *)backupKey +- (nullable NSData *)backupEncryptionKey { - return self.backupPrivateKey; + // TODO: Use actual encryption key. + return [@"temp" dataUsingEncoding:NSUTF8StringEncoding]; } - (void)backupJobDidSucceed:(OWSBackupJob *)backupJob diff --git a/Signal/src/util/OWSBackupAPI.swift b/Signal/src/util/OWSBackupAPI.swift index 79344a34e1..be943ff958 100644 --- a/Signal/src/util/OWSBackupAPI.swift +++ b/Signal/src/util/OWSBackupAPI.swift @@ -225,28 +225,27 @@ import CloudKit // MARK: - Delete @objc - public class func deleteRecordFromCloud(recordName: String, + public class func deleteRecordsFromCloud(recordNames: [String], success: @escaping (()) -> Void, failure: @escaping (Error) -> Void) { - deleteRecordFromCloud(recordName: recordName, + deleteRecordsFromCloud(recordNames: recordNames, remainingRetries: maxRetries, success: success, failure: failure) } - private class func deleteRecordFromCloud(recordName: String, + private class func deleteRecordsFromCloud(recordNames: [String], remainingRetries: Int, success: @escaping (()) -> Void, failure: @escaping (Error) -> Void) { - let recordID = CKRecordID(recordName: recordName) - - database().delete(withRecordID: recordID) { - (_, error) in + let recordIDs = recordNames.map { CKRecordID(recordName: $0) } + let deleteOperation = CKModifyRecordsOperation(recordsToSave: nil, recordIDsToDelete: recordIDs) + deleteOperation.modifyRecordsCompletionBlock = { (records, recordIds, error) in let outcome = outcomeForCloudKitError(error: error, - remainingRetries: remainingRetries, - label: "Delete Record") + remainingRetries: remainingRetries, + label: "Delete Records") switch outcome { case .success: success() @@ -254,23 +253,24 @@ import CloudKit failure(outcomeError) case .failureRetryAfterDelay(let retryDelay): DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: { - deleteRecordFromCloud(recordName: recordName, - remainingRetries: remainingRetries - 1, - success: success, - failure: failure) + deleteRecordsFromCloud(recordNames: recordNames, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) }) case .failureRetryWithoutDelay: DispatchQueue.global().async { - deleteRecordFromCloud(recordName: recordName, - remainingRetries: remainingRetries - 1, - success: success, - failure: failure) + deleteRecordsFromCloud(recordNames: recordNames, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) } case .unknownItem: owsFail("\(self.logTag) unexpected CloudKit response.") failure(invalidServiceResponseError()) } } + database().add(deleteOperation) } // MARK: - Exists? diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index 3a831c4cb5..f96e5585dd 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -3,12 +3,13 @@ // #import "OWSBackupExportJob.h" +#import "OWSBackupIO.h" #import "OWSDatabaseMigration.h" +#import "OWSSignalServiceProtos.pb.h" #import "Signal-Swift.h" #import #import #import -#import #import #import #import @@ -16,21 +17,192 @@ #import #import #import -#import -#import NS_ASSUME_NONNULL_BEGIN -NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKeySpec"; +@class OWSAttachmentExport; +@interface OWSBackupExportItem : NSObject + +@property (nonatomic) OWSBackupEncryptedItem *encryptedItem; + +@property (nonatomic) NSString *recordName; + +// This property is optional and is only used for attachments. +@property (nonatomic, nullable) OWSAttachmentExport *attachmentExport; + +// This property is optional. +// +// See comments in `OWSBackupIO`. +@property (nonatomic, nullable) NSNumber *uncompressedDataLength; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +#pragma mark - + +@implementation OWSBackupExportItem + +- (instancetype)initWithEncryptedItem:(OWSBackupEncryptedItem *)encryptedItem +{ + if (!(self = [super init])) { + return self; + } + + OWSAssert(encryptedItem); + + self.encryptedItem = encryptedItem; + + return self; +} + +@end + +#pragma mark - + +// Used to serialize database snapshot contents. +// Writes db entities using protobufs into snapshot fragments. +// Snapshot fragments are compressed (they compress _very well_, +// around 20x smaller) then encrypted. Ordering matters in +// snapshot contents (entities should be restored in the same +// order they are serialized), so we are always careful to preserve +// ordering of entities within a snapshot AND ordering of snapshot +// fragments within a bakckup. +// +// This stream is used to write entities one at a time and takes +// care of sharding them into fragments, compressing and encrypting +// those fragments. Fragment size is fixed to reduce worst case +// memory usage. +@interface OWSDBExportStream : NSObject + +@property (nonatomic) OWSBackupIO *backupIO; + +@property (nonatomic) NSMutableArray *exportItems; + +@property (nonatomic, nullable) OWSSignaliOSProtosBackupSnapshotBuilder *backupSnapshotBuilder; + +@property (nonatomic) NSUInteger cachedItemCount; + +@property (nonatomic) NSUInteger totalItemCount; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +#pragma mark - + +@implementation OWSDBExportStream + +- (instancetype)initWithBackupIO:(OWSBackupIO *)backupIO +{ + if (!(self = [super init])) { + return self; + } + + OWSAssert(backupIO); + + self.exportItems = [NSMutableArray new]; + self.backupIO = backupIO; + + return self; +} + + +// It isn't strictly necessary to capture the entity type (the importer doesn't +// use this state), but I think it'll be helpful to have around to future-proof +// this work, help with debugging issue, etc. +- (BOOL)writeObject:(TSYapDatabaseObject *)object + entityType:(OWSSignaliOSProtosBackupSnapshotBackupEntityType)entityType +{ + OWSAssert(object); + + NSData *_Nullable data = [NSKeyedArchiver archivedDataWithRootObject:object]; + if (!data) { + OWSProdLogAndFail(@"%@ couldn't serialize database object: %@", self.logTag, [object class]); + return NO; + } + + if (!self.backupSnapshotBuilder) { + self.backupSnapshotBuilder = [OWSSignaliOSProtosBackupSnapshotBuilder new]; + } + + OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder *entityBuilder = + [OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder new]; + [entityBuilder setType:entityType]; + [entityBuilder setEntityData:data]; + + [self.backupSnapshotBuilder addEntity:[entityBuilder build]]; + + self.cachedItemCount = self.cachedItemCount + 1; + self.totalItemCount = self.totalItemCount + 1; + + static const int kMaxDBSnapshotSize = 1000; + if (self.cachedItemCount > kMaxDBSnapshotSize) { + @autoreleasepool { + return [self flush]; + } + } + + return YES; +} + +// Write cached data to disk, if necessary. +// +// Returns YES on success. +- (BOOL)flush +{ + if (!self.backupSnapshotBuilder) { + // No data to flush to disk. + return YES; + } + + // Try to release allocated buffers ASAP. + @autoreleasepool { + NSData *_Nullable uncompressedData = [self.backupSnapshotBuilder build].data; + NSUInteger uncompressedDataLength = uncompressedData.length; + self.backupSnapshotBuilder = nil; + self.cachedItemCount = 0; + if (!uncompressedData) { + OWSProdLogAndFail(@"%@ couldn't convert database snapshot to data.", self.logTag); + return NO; + } + + NSData *compressedData = [self.backupIO compressData:uncompressedData]; + + OWSBackupEncryptedItem *_Nullable encryptedItem = [self.backupIO encryptDataAsTempFile:compressedData]; + if (!encryptedItem) { + OWSProdLogAndFail(@"%@ couldn't encrypt database snapshot.", self.logTag); + return NO; + } + + OWSBackupExportItem *exportItem = [[OWSBackupExportItem alloc] initWithEncryptedItem:encryptedItem]; + exportItem.uncompressedDataLength = @(uncompressedDataLength); + [self.exportItems addObject:exportItem]; + } + + return YES; +} + +@end + +#pragma mark - + +// This class is used to: +// +// * Lazy-encrypt and eagerly cleanup attachment uploads. +// To reduce disk footprint of backup export process, +// we only want to have one attachment export on disk +// at a time. @interface OWSAttachmentExport : NSObject -@property (nonatomic, weak) id delegate; -@property (nonatomic) NSString *jobTempDirPath; +@property (nonatomic) OWSBackupIO *backupIO; @property (nonatomic) NSString *attachmentId; @property (nonatomic) NSString *attachmentFilePath; -@property (nonatomic, nullable) NSString *tempFilePath; @property (nonatomic, nullable) NSString *relativeFilePath; +@property (nonatomic) OWSBackupEncryptedItem *encryptedItem; + +- (instancetype)init NS_UNAVAILABLE; @end @@ -38,21 +210,38 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe @implementation OWSAttachmentExport +- (instancetype)initWithBackupIO:(OWSBackupIO *)backupIO + attachmentId:(NSString *)attachmentId + attachmentFilePath:(NSString *)attachmentFilePath +{ + if (!(self = [super init])) { + return self; + } + + OWSAssert(backupIO); + OWSAssert(attachmentId.length > 0); + OWSAssert(attachmentFilePath.length > 0); + + self.backupIO = backupIO; + self.attachmentId = attachmentId; + self.attachmentFilePath = attachmentFilePath; + + return self; +} + - (void)dealloc { // Surface memory leaks by logging the deallocation. DDLogVerbose(@"Dealloc: %@", self.class); - // Delete temporary file ASAP. - if (self.tempFilePath) { - [OWSFileSystem deleteFileIfExists:self.tempFilePath]; - } + [self cleanUp]; } -// On success, tempFilePath will be non-nil. -- (void)prepareForUpload +// On success, encryptedItem will be non-nil. +// +// Returns YES on success. +- (BOOL)prepareForUpload { - OWSAssert(self.jobTempDirPath.length > 0); OWSAssert(self.attachmentId.length > 0); OWSAssert(self.attachmentFilePath.length > 0); @@ -60,7 +249,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe if (![self.attachmentFilePath hasPrefix:attachmentsDirPath]) { DDLogError(@"%@ attachment has unexpected path.", self.logTag); OWSFail(@"%@ attachment has unexpected path: %@", self.logTag, self.attachmentFilePath); - return; + return NO; } NSString *relativeFilePath = [self.attachmentFilePath substringFromIndex:attachmentsDirPath.length]; NSString *pathSeparator = @"/"; @@ -69,15 +258,20 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe } self.relativeFilePath = relativeFilePath; - NSString *_Nullable tempFilePath = [OWSBackupExportJob encryptFileAsTempFile:self.attachmentFilePath - jobTempDirPath:self.jobTempDirPath - delegate:self.delegate]; - if (!tempFilePath) { + OWSBackupEncryptedItem *_Nullable encryptedItem = [self.backupIO encryptFileAsTempFile:self.attachmentFilePath]; + if (!encryptedItem) { DDLogError(@"%@ attachment could not be encrypted.", self.logTag); OWSFail(@"%@ attachment could not be encrypted: %@", self.logTag, self.attachmentFilePath); - return; + return NO; } - self.tempFilePath = tempFilePath; + self.encryptedItem = encryptedItem; + return YES; +} + +// Returns YES on success. +- (BOOL)cleanUp +{ + return [OWSFileSystem deleteFileIfExists:self.encryptedItem.filePath]; } @end @@ -88,19 +282,17 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe @property (nonatomic, nullable) OWSBackgroundTask *backgroundTask; -@property (nonatomic, nullable) OWSBackupStorage *backupStorage; +@property (nonatomic) OWSBackupIO *backupIO; -@property (nonatomic) NSMutableArray *databaseFilePaths; -// A map of "record name"-to-"file name". -@property (nonatomic) NSMutableDictionary *databaseRecordMap; +@property (nonatomic) NSMutableArray *unsavedDatabaseItems; -// A map of "attachment id"-to-"local file path". -@property (nonatomic) NSMutableDictionary *attachmentFilePathMap; -// A map of "record name"-to-"file relative path". -@property (nonatomic) NSMutableDictionary *attachmentRecordMap; +@property (nonatomic) NSMutableArray *unsavedAttachmentExports; -@property (nonatomic, nullable) NSString *manifestFilePath; -@property (nonatomic, nullable) NSString *manifestRecordName; +@property (nonatomic) NSMutableArray *savedDatabaseItems; + +@property (nonatomic) NSMutableArray *savedAttachmentItems; + +@property (nonatomic, nullable) OWSBackupExportItem *manifestItem; @end @@ -135,7 +327,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe progress:nil]; __weak OWSBackupExportJob *weakSelf = self; - [self configureExport:^(BOOL configureExportSuccess) { + [self configureExportWithCompletion:^(BOOL configureExportSuccess) { if (!configureExportSuccess) { [self failWithErrorDescription: NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", @@ -149,35 +341,32 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_EXPORT", @"Indicates that the backup export data is being exported.") progress:nil]; - [self exportDatabase:^(BOOL exportDatabaseSuccess) { - if (!exportDatabaseSuccess) { - [self failWithErrorDescription: - NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", - @"Error indicating the a backup export could not export the user's data.")]; + if (![self exportDatabase]) { + [self failWithErrorDescription: + NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", + @"Error indicating the a backup export could not export the user's data.")]; + return; + } + if (self.isComplete) { + return; + } + [self saveToCloudWithCompletion:^(NSError *_Nullable saveError) { + if (saveError) { + [weakSelf failWithError:saveError]; return; } - - if (self.isComplete) { - return; - } - [self saveToCloud:^(NSError *_Nullable saveError) { - if (saveError) { - [weakSelf failWithError:saveError]; + [self cleanUpCloudWithCompletion:^(NSError *_Nullable cleanUpError) { + if (cleanUpError) { + [weakSelf failWithError:cleanUpError]; return; } - [self cleanUpCloud:^(NSError *_Nullable cleanUpError) { - if (cleanUpError) { - [weakSelf failWithError:cleanUpError]; - return; - } - [weakSelf succeed]; - }]; + [weakSelf succeed]; }]; }]; }]; } -- (void)configureExport:(OWSBackupJobBoolCompletion)completion +- (void)configureExportWithCompletion:(OWSBackupJobBoolCompletion)completion { OWSAssert(completion); @@ -188,6 +377,8 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe return completion(NO); } + self.backupIO = [[OWSBackupIO alloc] initWithJobTempDirPath:self.jobTempDirPath]; + // We need to verify that we have a valid account. // Otherwise, if we re-register on another device, we // continue to backup on our old device, overwriting @@ -211,223 +402,210 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe }]; } -- (void)exportDatabase:(OWSBackupJobBoolCompletion)completion +- (BOOL)exportDatabase { - OWSAssert(completion); + OWSAssert(self.backupIO); DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - if (![OWSBackupJob generateRandomDatabaseKeySpecWithKeychainKey:kOWSBackup_ExportDatabaseKeySpec]) { - OWSProdLogAndFail(@"%@ Could not generate database key spec for export.", self.logTag); - return completion(NO); + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_DATABASE_EXPORT", + @"Indicates that the database data is being exported.") + progress:nil]; + + YapDatabaseConnection *_Nullable dbConnection = self.primaryStorage.newDatabaseConnection; + if (!dbConnection) { + OWSProdLogAndFail(@"%@ Could not create dbConnection.", self.logTag); + return NO; } - // TODO: Move this work into the "export database" method. - NSString *jobDatabaseDirPath = [self.jobTempDirPath stringByAppendingPathComponent:@"database"]; - if (![OWSFileSystem ensureDirectoryExists:jobDatabaseDirPath]) { - OWSProdLogAndFail(@"%@ Could not create jobDatabaseDirPath.", self.logTag); - return completion(NO); - } + OWSDBExportStream *exportStream = [[OWSDBExportStream alloc] initWithBackupIO:self.backupIO]; - BackupStorageKeySpecBlock keySpecBlock = ^{ - NSData *_Nullable databaseKeySpec = - [OWSBackupJob loadDatabaseKeySpecWithKeychainKey:kOWSBackup_ExportDatabaseKeySpec]; - if (!databaseKeySpec) { - OWSProdLogAndFail(@"%@ Could not load database keyspec for export.", self.logTag); - } - return databaseKeySpec; + __block BOOL aborted = NO; + typedef BOOL (^EntityFilter)(id object); + typedef NSUInteger (^ExportBlock)(YapDatabaseReadTransaction *, + NSString *, + Class, + EntityFilter _Nullable, + OWSSignaliOSProtosBackupSnapshotBackupEntityType); + ExportBlock exportEntities = ^(YapDatabaseReadTransaction *transaction, + NSString *collection, + Class expectedClass, + EntityFilter _Nullable filter, + OWSSignaliOSProtosBackupSnapshotBackupEntityType entityType) { + __block NSUInteger count = 0; + [transaction + enumerateKeysAndObjectsInCollection:collection + usingBlock:^(NSString *key, id object, BOOL *stop) { + if (self.isComplete) { + *stop = YES; + return; + } + if (filter && !filter(object)) { + return; + } + if (![object isKindOfClass:expectedClass]) { + OWSProdLogAndFail(@"%@ unexpected class: %@", self.logTag, [object class]); + return; + } + TSYapDatabaseObject *entity = object; + count++; + + if (![exportStream writeObject:entity entityType:entityType]) { + *stop = YES; + aborted = YES; + return; + } + }]; + return count; }; - self.backupStorage = - [[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock]; - if (!self.backupStorage) { - OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag); - return completion(NO); - } + __block NSUInteger copiedThreads = 0; + __block NSUInteger copiedInteractions = 0; + __block NSUInteger copiedAttachments = 0; + __block NSUInteger copiedMigrations = 0; + self.unsavedAttachmentExports = [NSMutableArray new]; + [dbConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) { + copiedThreads = exportEntities(transaction, + [TSThread collection], + [TSThread class], + nil, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeThread); + if (aborted) { + return; + } - // TODO: Do we really need to run these registrations on the main thread? - __weak OWSBackupExportJob *weakSelf = self; - dispatch_async(dispatch_get_main_queue(), ^{ - [weakSelf.backupStorage runSyncRegistrations]; - [weakSelf.backupStorage runAsyncRegistrationsWithCompletion:^{ - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - completion([weakSelf exportDatabaseContents]); - }); - }]; - }); -} + copiedAttachments = exportEntities(transaction, + [TSAttachment collection], + [TSAttachment class], + ^(id object) { + if (![object isKindOfClass:[TSAttachmentStream class]]) { + return NO; + } + TSAttachmentStream *attachmentStream = object; + NSString *_Nullable filePath = attachmentStream.filePath; + if (!filePath) { + DDLogError(@"%@ attachment is missing file.", self.logTag); + return NO; + OWSAssert(attachmentStream.uniqueId.length > 0); + } -- (BOOL)exportDatabaseContents -{ - DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + // OWSAttachmentExport is used to lazily write an encrypted copy of the + // attachment to disk. + OWSAttachmentExport *attachmentExport = + [[OWSAttachmentExport alloc] initWithBackupIO:self.backupIO + attachmentId:attachmentStream.uniqueId + attachmentFilePath:filePath]; + [self.unsavedAttachmentExports addObject:attachmentExport]; - YapDatabaseConnection *_Nullable tempDBConnection = self.backupStorage.newDatabaseConnection; - if (!tempDBConnection) { - OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag); - return NO; - } - YapDatabaseConnection *_Nullable primaryDBConnection = self.primaryStorage.newDatabaseConnection; - if (!primaryDBConnection) { - OWSProdLogAndFail(@"%@ Could not create primaryDBConnection.", self.logTag); - return NO; - } + return YES; + }, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeAttachment); + if (aborted) { + return; + } - __block unsigned long long copiedThreads = 0; - __block unsigned long long copiedInteractions = 0; - __block unsigned long long copiedEntities = 0; - __block unsigned long long copiedAttachments = 0; - __block unsigned long long copiedMigrations = 0; + // Interactions refer to threads and attachments, so copy after them. + copiedInteractions = exportEntities(transaction, + [TSInteraction collection], + [TSInteraction class], + ^(id object) { + // Ignore disappearing messages. + if ([object isKindOfClass:[TSMessage class]]) { + TSMessage *message = object; + if (message.isExpiringMessage) { + return NO; + } + } + TSInteraction *interaction = object; + // Ignore dynamic interactions. + if (interaction.isDynamicInteraction) { + return NO; + } + return YES; + }, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeInteraction); + if (aborted) { + return; + } - self.attachmentFilePathMap = [NSMutableDictionary new]; - - [primaryDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) { - [tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { - [dstTransaction setObject:@(YES) - forKey:kOWSBackup_Snapshot_ValidKey - inCollection:kOWSBackup_Snapshot_Collection]; - - // Copy threads. - [srcTransaction - enumerateKeysAndObjectsInCollection:[TSThread collection] - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - return; - } - if (![object isKindOfClass:[TSThread class]]) { - OWSProdLogAndFail( - @"%@ unexpected class: %@", self.logTag, [object class]); - return; - } - TSThread *thread = object; - [thread saveWithTransaction:dstTransaction]; - copiedThreads++; - copiedEntities++; - }]; - - // Copy attachments. - [srcTransaction - enumerateKeysAndObjectsInCollection:[TSAttachment collection] - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - return; - } - if (![object isKindOfClass:[TSAttachment class]]) { - OWSProdLogAndFail( - @"%@ unexpected class: %@", self.logTag, [object class]); - return; - } - if ([object isKindOfClass:[TSAttachmentStream class]]) { - TSAttachmentStream *attachmentStream = object; - NSString *_Nullable filePath = attachmentStream.filePath; - if (filePath) { - OWSAssert(attachmentStream.uniqueId.length > 0); - self.attachmentFilePathMap[attachmentStream.uniqueId] = filePath; - } - } - TSAttachment *attachment = object; - [attachment saveWithTransaction:dstTransaction]; - copiedAttachments++; - copiedEntities++; - }]; - - // Copy interactions. - // - // Interactions refer to threads and attachments, so copy the last. - [srcTransaction - enumerateKeysAndObjectsInCollection:[TSInteraction collection] - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - return; - } - if (![object isKindOfClass:[TSInteraction class]]) { - OWSProdLogAndFail( - @"%@ unexpected class: %@", self.logTag, [object class]); - return; - } - // Ignore disappearing messages. - if ([object isKindOfClass:[TSMessage class]]) { - TSMessage *message = object; - if (message.isExpiringMessage) { - return; - } - } - TSInteraction *interaction = object; - // Ignore dynamic interactions. - if (interaction.isDynamicInteraction) { - return; - } - [interaction saveWithTransaction:dstTransaction]; - copiedInteractions++; - copiedEntities++; - }]; - - // Copy migrations. - [srcTransaction - enumerateKeysAndObjectsInCollection:[OWSDatabaseMigration collection] - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - return; - } - if (![object isKindOfClass:[OWSDatabaseMigration class]]) { - OWSProdLogAndFail( - @"%@ unexpected class: %@", self.logTag, [object class]); - return; - } - OWSDatabaseMigration *migration = object; - [migration saveWithTransaction:dstTransaction]; - copiedMigrations++; - copiedEntities++; - }]; - }]; + copiedMigrations = exportEntities(transaction, + [OWSDatabaseMigration collection], + [OWSDatabaseMigration class], + nil, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeMigration); }]; - if (self.isComplete) { + if (aborted || self.isComplete) { return NO; } + + @autoreleasepool { + if (![exportStream flush]) { + OWSProdLogAndFail(@"%@ Could not flush database snapshots.", self.logTag); + return NO; + } + } + + self.unsavedDatabaseItems = [exportStream.exportItems mutableCopy]; + // TODO: Should we do a database checkpoint? - DDLogInfo(@"%@ copiedThreads: %llu", self.logTag, copiedThreads); - DDLogInfo(@"%@ copiedMessages: %llu", self.logTag, copiedInteractions); - DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities); - DDLogInfo(@"%@ copiedAttachments: %llu", self.logTag, copiedAttachments); - DDLogInfo(@"%@ copiedMigrations: %llu", self.logTag, copiedMigrations); - - [self.backupStorage logFileSizes]; - - // Capture the list of files to save. - self.databaseFilePaths = [@[ - self.backupStorage.databaseFilePath, - self.backupStorage.databaseFilePath_WAL, - self.backupStorage.databaseFilePath_SHM, - ] mutableCopy]; - - // Close the database. - tempDBConnection = nil; - self.backupStorage = nil; + DDLogInfo(@"%@ copiedThreads: %zd", self.logTag, copiedThreads); + DDLogInfo(@"%@ copiedMessages: %zd", self.logTag, copiedInteractions); + DDLogInfo(@"%@ copiedAttachments: %zd", self.logTag, copiedAttachments); + DDLogInfo(@"%@ copiedMigrations: %zd", self.logTag, copiedMigrations); + DDLogInfo(@"%@ copiedEntities: %zd", self.logTag, exportStream.totalItemCount); return YES; } -- (void)saveToCloud:(OWSBackupJobCompletion)completion +- (void)saveToCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - self.databaseRecordMap = [NSMutableDictionary new]; - self.attachmentRecordMap = [NSMutableDictionary new]; + self.savedDatabaseItems = [NSMutableArray new]; + self.savedAttachmentItems = [NSMutableArray new]; - [self saveNextFileToCloud:completion]; + unsigned long long totalFileSize = 0; + NSUInteger totalFileCount = 0; + { + unsigned long long databaseFileSize = 0; + for (OWSBackupExportItem *item in self.unsavedDatabaseItems) { + databaseFileSize += [OWSFileSystem fileSizeOfPath:item.encryptedItem.filePath].unsignedLongLongValue; + } + DDLogInfo(@"%@ exporting %@: count: %zd, bytes: %llu.", + self.logTag, + @"database items", + self.unsavedDatabaseItems.count, + databaseFileSize); + totalFileSize += databaseFileSize; + totalFileCount += self.unsavedDatabaseItems.count; + } + { + unsigned long long attachmentFileSize = 0; + for (OWSAttachmentExport *attachmentExport in self.unsavedAttachmentExports) { + attachmentFileSize += + [OWSFileSystem fileSizeOfPath:attachmentExport.attachmentFilePath].unsignedLongLongValue; + } + DDLogInfo(@"%@ exporting %@: count: %zd, bytes: %llu.", + self.logTag, + @"attachment items", + self.unsavedAttachmentExports.count, + attachmentFileSize); + totalFileSize += attachmentFileSize; + totalFileCount += self.unsavedAttachmentExports.count; + } + DDLogInfo(@"%@ exporting %@: count: %zd, bytes: %llu.", self.logTag, @"all items", totalFileCount, totalFileSize); + + [self saveNextFileToCloudWithCompletion:completion]; } // This method uploads one file (the "next" file) each time it // is called. Each successful file upload re-invokes this method // until the last (the manifest file). -- (void)saveNextFileToCloud:(OWSBackupJobCompletion)completion +- (void)saveNextFileToCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); @@ -435,78 +613,86 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe return; } - CGFloat progress - = (self.databaseRecordMap.count / (CGFloat)(self.databaseRecordMap.count + self.databaseFilePaths.count)); + // Add one for the manifest + NSUInteger unsavedCount = (self.unsavedDatabaseItems.count + self.unsavedAttachmentExports.count + 1); + NSUInteger savedCount = (self.savedDatabaseItems.count + self.savedAttachmentItems.count); + + CGFloat progress = (savedCount / (CGFloat)(unsavedCount + savedCount)); [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_UPLOAD", @"Indicates that the backup export data is being uploaded.") progress:@(progress)]; - if ([self saveNextDatabaseFileToCloud:completion]) { + if ([self saveNextDatabaseFileToCloudWithCompletion:completion]) { return; } - if ([self saveNextAttachmentFileToCloud:completion]) { + if ([self saveNextAttachmentFileToCloudWithCompletion:completion]) { return; } - [self saveManifestFileToCloud:completion]; + [self saveManifestFileToCloudWithCompletion:completion]; } -- (BOOL)saveNextDatabaseFileToCloud:(OWSBackupJobCompletion)completion +// This method returns YES IFF "work was done and there might be more work to do". +- (BOOL)saveNextDatabaseFileToCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); __weak OWSBackupExportJob *weakSelf = self; - if (self.databaseFilePaths.count < 1) { + if (self.unsavedDatabaseItems.count < 1) { return NO; } - NSString *filePath = self.databaseFilePaths.lastObject; - [self.databaseFilePaths removeLastObject]; - // Database files are encrypted and can be safely stored unencrypted in the cloud. - // TODO: Security review. - [OWSBackupAPI saveEphemeralDatabaseFileToCloudWithFileUrl:[NSURL fileURLWithPath:filePath] + // Pop next item from queue, preserving ordering. + OWSBackupExportItem *item = self.unsavedDatabaseItems.firstObject; + [self.unsavedDatabaseItems removeObjectAtIndex:0]; + + OWSAssert(item.encryptedItem.filePath.length > 0); + + [OWSBackupAPI saveEphemeralDatabaseFileToCloudWithFileUrl:[NSURL fileURLWithPath:item.encryptedItem.filePath] success:^(NSString *recordName) { // Ensure that we continue to work off the main thread. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - OWSBackupExportJob *strongSelf = weakSelf; - if (!strongSelf) { - return; - } - strongSelf.databaseRecordMap[recordName] = [filePath lastPathComponent]; - [strongSelf saveNextFileToCloud:completion]; + item.recordName = recordName; + [weakSelf.savedDatabaseItems addObject:item]; + [weakSelf saveNextFileToCloudWithCompletion:completion]; }); } failure:^(NSError *error) { // Database files are critical so any error uploading them is unrecoverable. + DDLogVerbose(@"%@ error while saving file: %@", weakSelf.logTag, item.encryptedItem.filePath); completion(error); }]; return YES; } -- (BOOL)saveNextAttachmentFileToCloud:(OWSBackupJobCompletion)completion +// This method returns YES IFF "work was done and there might be more work to do". +- (BOOL)saveNextAttachmentFileToCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); __weak OWSBackupExportJob *weakSelf = self; - if (self.attachmentFilePathMap.count < 1) { + if (self.unsavedAttachmentExports.count < 1) { return NO; } - NSString *attachmentId = self.attachmentFilePathMap.allKeys.lastObject; - NSString *attachmentFilePath = self.attachmentFilePathMap[attachmentId]; - [self.attachmentFilePathMap removeObjectForKey:attachmentId]; + // No need to preserve ordering of attachments. + OWSAttachmentExport *attachmentExport = self.unsavedAttachmentExports.lastObject; + [self.unsavedAttachmentExports removeLastObject]; - // OWSAttachmentExport is used to lazily write an encrypted copy of the - // attachment to disk. - OWSAttachmentExport *attachmentExport = [OWSAttachmentExport new]; - attachmentExport.delegate = self.delegate; - attachmentExport.jobTempDirPath = self.jobTempDirPath; - attachmentExport.attachmentId = attachmentId; - attachmentExport.attachmentFilePath = attachmentFilePath; + @autoreleasepool { + // OWSAttachmentExport is used to lazily write an encrypted copy of the + // attachment to disk. + if (![attachmentExport prepareForUpload]) { + // Attachment files are non-critical so any error uploading them is recoverable. + [weakSelf saveNextFileToCloudWithCompletion:completion]; + return YES; + } + OWSAssert(attachmentExport.relativeFilePath.length > 0); + OWSAssert(attachmentExport.encryptedItem); + } - [OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentId + [OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentExport.attachmentId fileUrlBlock:^{ - [attachmentExport prepareForUpload]; - if (attachmentExport.tempFilePath.length < 1) { + if (attachmentExport.encryptedItem.filePath.length < 1) { DDLogError(@"%@ attachment export missing temp file path", self.logTag); return (NSURL *)nil; } @@ -514,7 +700,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe DDLogError(@"%@ attachment export missing relative file path", self.logTag); return (NSURL *)nil; } - return [NSURL fileURLWithPath:attachmentExport.tempFilePath]; + return [NSURL fileURLWithPath:attachmentExport.encryptedItem.filePath]; } success:^(NSString *recordName) { // Ensure that we continue to work off the main thread. @@ -523,39 +709,59 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe if (!strongSelf) { return; } - strongSelf.attachmentRecordMap[recordName] = attachmentExport.relativeFilePath; - DDLogVerbose(@"%@ exported attachment: %@ as %@", + + if (![attachmentExport cleanUp]) { + DDLogError(@"%@ couldn't clean up attachment export.", self.logTag); + // Attachment files are non-critical so any error uploading them is recoverable. + } + + OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; + exportItem.encryptedItem = attachmentExport.encryptedItem; + exportItem.recordName = recordName; + exportItem.attachmentExport = attachmentExport; + [strongSelf.savedAttachmentItems addObject:exportItem]; + + DDLogVerbose(@"%@ saved attachment: %@ as %@", self.logTag, - attachmentFilePath, + attachmentExport.attachmentFilePath, attachmentExport.relativeFilePath); - [strongSelf saveNextFileToCloud:completion]; + [strongSelf saveNextFileToCloudWithCompletion:completion]; }); } failure:^(NSError *error) { // Ensure that we continue to work off the main thread. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + if (![attachmentExport cleanUp]) { + DDLogError(@"%@ couldn't clean up attachment export.", self.logTag); + // Attachment files are non-critical so any error uploading them is recoverable. + } + // Attachment files are non-critical so any error uploading them is recoverable. - [weakSelf saveNextFileToCloud:completion]; + [weakSelf saveNextFileToCloudWithCompletion:completion]; }); }]; + return YES; } -- (void)saveManifestFileToCloud:(OWSBackupJobCompletion)completion +- (void)saveManifestFileToCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); - if (![self writeManifestFile]) { + OWSBackupEncryptedItem *_Nullable encryptedItem = [self writeManifestFile]; + if (!encryptedItem) { completion(OWSErrorWithCodeDescription(OWSErrorCodeExportBackupFailed, NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", @"Error indicating the a backup export could not export the user's data."))); return; } - OWSAssert(self.manifestFilePath); + + OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; + exportItem.encryptedItem = encryptedItem; __weak OWSBackupExportJob *weakSelf = self; - [OWSBackupAPI upsertManifestFileToCloudWithFileUrl:[NSURL fileURLWithPath:self.manifestFilePath] + [OWSBackupAPI upsertManifestFileToCloudWithFileUrl:[NSURL fileURLWithPath:encryptedItem.filePath] success:^(NSString *recordName) { // Ensure that we continue to work off the main thread. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ @@ -563,7 +769,9 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe if (!strongSelf) { return; } - strongSelf.manifestRecordName = recordName; + + exportItem.recordName = recordName; + strongSelf.manifestItem = exportItem; // All files have been saved to the cloud. completion(nil); @@ -575,24 +783,16 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe }]; } -- (BOOL)writeManifestFile +- (nullable OWSBackupEncryptedItem *)writeManifestFile { - OWSAssert(self.databaseRecordMap.count > 0); - OWSAssert(self.attachmentRecordMap); + OWSAssert(self.savedDatabaseItems.count > 0); + OWSAssert(self.savedAttachmentItems); OWSAssert(self.jobTempDirPath.length > 0); - - NSData *_Nullable databaseKeySpec = - [OWSBackupJob loadDatabaseKeySpecWithKeychainKey:kOWSBackup_ExportDatabaseKeySpec]; - if (databaseKeySpec.length < 1) { - OWSProdLogAndFail(@"%@ Could not load database keyspec for export.", self.logTag); - return nil; - } + OWSAssert(self.backupIO); NSDictionary *json = @{ - kOWSBackup_ManifestKey_DatabaseFiles : self.databaseRecordMap, - kOWSBackup_ManifestKey_AttachmentFiles : self.attachmentRecordMap, - // JSON doesn't support byte arrays. - kOWSBackup_ManifestKey_DatabaseKeySpec : databaseKeySpec.base64EncodedString, + kOWSBackup_ManifestKey_DatabaseFiles : [self jsonForItems:self.savedDatabaseItems], + kOWSBackup_ManifestKey_AttachmentFiles : [self jsonForItems:self.savedAttachmentItems], }; DDLogVerbose(@"%@ json: %@", self.logTag, json); @@ -602,14 +802,36 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [NSJSONSerialization dataWithJSONObject:json options:NSJSONWritingPrettyPrinted error:&error]; if (!jsonData || error) { OWSProdLogAndFail(@"%@ error encoding manifest file: %@", self.logTag, error); - return NO; + return nil; } - self.manifestFilePath = - [OWSBackupJob encryptDataAsTempFile:jsonData jobTempDirPath:self.jobTempDirPath delegate:self.delegate]; - return self.manifestFilePath != nil; + return [self.backupIO encryptDataAsTempFile:jsonData encryptionKey:self.delegate.backupEncryptionKey]; } -- (void)cleanUpCloud:(OWSBackupJobCompletion)completion +- (NSArray *> *)jsonForItems:(NSArray *)items +{ + NSMutableArray *result = [NSMutableArray new]; + for (OWSBackupExportItem *item in items) { + NSMutableDictionary *itemJson = [NSMutableDictionary new]; + OWSAssert(item.recordName.length > 0); + + itemJson[kOWSBackup_ManifestKey_RecordName] = item.recordName; + OWSAssert(item.encryptedItem.filePath.length > 0); + OWSAssert(item.encryptedItem.encryptionKey.length > 0); + itemJson[kOWSBackup_ManifestKey_EncryptionKey] = item.encryptedItem.encryptionKey.base64EncodedString; + if (item.attachmentExport) { + OWSAssert(item.attachmentExport.relativeFilePath.length > 0); + itemJson[kOWSBackup_ManifestKey_RelativeFilePath] = item.attachmentExport.relativeFilePath; + } + if (item.uncompressedDataLength) { + itemJson[kOWSBackup_ManifestKey_DataSize] = item.uncompressedDataLength; + } + [result addObject:itemJson]; + } + + return result; +} + +- (void)cleanUpCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); @@ -624,14 +846,24 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe // records not involved in this backup export. NSMutableSet *activeRecordNames = [NSMutableSet new]; - OWSAssert(self.databaseRecordMap.count > 0); - [activeRecordNames addObjectsFromArray:self.databaseRecordMap.allKeys]; + OWSAssert(self.savedDatabaseItems.count > 0); + for (OWSBackupExportItem *item in self.savedDatabaseItems) { + OWSAssert(item.recordName.length > 0); + OWSAssert(![activeRecordNames containsObject:item.recordName]); + [activeRecordNames addObject:item.recordName]; + } + for (OWSBackupExportItem *item in self.savedAttachmentItems) { + OWSAssert(item.recordName.length > 0); + OWSAssert(![activeRecordNames containsObject:item.recordName]); + [activeRecordNames addObject:item.recordName]; + } + OWSAssert(self.manifestItem.recordName.length > 0); + OWSAssert(![activeRecordNames containsObject:self.manifestItem.recordName]); + [activeRecordNames addObject:self.manifestItem.recordName]; - OWSAssert(self.attachmentRecordMap); - [activeRecordNames addObjectsFromArray:self.attachmentRecordMap.allKeys]; - - OWSAssert(self.manifestRecordName.length > 0); - [activeRecordNames addObject:self.manifestRecordName]; + // TODO: If we implement "lazy restores" where attachments (etc.) are + // restored lazily, we need to include the record names for all + // records that haven't been restored yet. __weak OWSBackupExportJob *weakSelf = self; [OWSBackupAPI fetchAllRecordNamesWithSuccess:^(NSArray *recordNames) { @@ -672,8 +904,12 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe if (obsoleteRecordNames.count < 1) { // No more records to delete; cleanup is complete. - completion(nil); - return; + return completion(nil); + } + + if (self.isComplete) { + // Job was aborted. + return completion(nil); } CGFloat progress = (obsoleteRecordNames.count / (CGFloat)(obsoleteRecordNames.count + deletedCount)); @@ -681,16 +917,21 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe @"Indicates that the cloud is being cleaned up.") progress:@(progress)]; - NSString *recordName = obsoleteRecordNames.lastObject; - [obsoleteRecordNames removeLastObject]; + static const NSUInteger kMaxBatchSize = 100; + NSMutableArray *batchRecordNames = [NSMutableArray new]; + while (obsoleteRecordNames.count > 0 && batchRecordNames.count < kMaxBatchSize) { + NSString *recordName = obsoleteRecordNames.lastObject; + [obsoleteRecordNames removeLastObject]; + [batchRecordNames addObject:recordName]; + } __weak OWSBackupExportJob *weakSelf = self; - [OWSBackupAPI deleteRecordFromCloudWithRecordName:recordName + [OWSBackupAPI deleteRecordsFromCloudWithRecordNames:batchRecordNames success:^{ // Ensure that we continue to work off the main thread. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ [weakSelf deleteRecordsFromCloud:obsoleteRecordNames - deletedCount:deletedCount + 1 + deletedCount:deletedCount + batchRecordNames.count completion:completion]; }); } @@ -699,7 +940,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ // Cloud cleanup is non-critical so any error is recoverable. [weakSelf deleteRecordsFromCloud:obsoleteRecordNames - deletedCount:deletedCount + 1 + deletedCount:deletedCount + batchRecordNames.count completion:completion]; }); }]; diff --git a/Signal/src/util/OWSBackupIO.h b/Signal/src/util/OWSBackupIO.h new file mode 100644 index 0000000000..fabb0088d3 --- /dev/null +++ b/Signal/src/util/OWSBackupIO.h @@ -0,0 +1,57 @@ +// +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. +// + +NS_ASSUME_NONNULL_BEGIN + +@interface OWSBackupEncryptedItem : NSObject + +@property (nonatomic) NSString *filePath; + +@property (nonatomic) NSData *encryptionKey; + +@end + +#pragma mark - + +@interface OWSBackupIO : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +- (instancetype)initWithJobTempDirPath:(NSString *)jobTempDirPath; + +#pragma mark - Encrypt + +- (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath; + +- (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath + encryptionKey:(NSData *)encryptionKey; + +- (nullable OWSBackupEncryptedItem *)encryptDataAsTempFile:(NSData *)srcData; + +- (nullable OWSBackupEncryptedItem *)encryptDataAsTempFile:(NSData *)srcData encryptionKey:(NSData *)encryptionKey; + +#pragma mark - Decrypt + +- (BOOL)decryptFileAsFile:(NSString *)srcFilePath + dstFilePath:(NSString *)dstFilePath + encryptionKey:(NSData *)encryptionKey; + +- (nullable NSData *)decryptFileAsData:(NSString *)srcFilePath encryptionKey:(NSData *)encryptionKey; + +- (nullable NSData *)decryptDataAsData:(NSData *)srcData encryptionKey:(NSData *)encryptionKey; + +#pragma mark - Compression + +- (nullable NSData *)compressData:(NSData *)srcData; + +// I'm using the (new in iOS 9) compressionlib. One of its weaknesses is that it +// requires you to pre-allocate output buffers during compression and decompression. +// During decompression this is particularly tricky since there's no way to safely +// predict how large the output will be based on the input. So, we store the +// uncompressed size for compressed backup items. +- (nullable NSData *)decompressData:(NSData *)srcData uncompressedDataLength:(NSUInteger)uncompressedDataLength; + +@end + +NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackupIO.m b/Signal/src/util/OWSBackupIO.m new file mode 100644 index 0000000000..02b50d9fb7 --- /dev/null +++ b/Signal/src/util/OWSBackupIO.m @@ -0,0 +1,255 @@ +// +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. +// + +#import "OWSBackupIO.h" +#import +#import + +@import Compression; + +NS_ASSUME_NONNULL_BEGIN + +// TODO: +static const NSUInteger kOWSBackupKeyLength = 32; + +// LZMA algorithm significantly outperforms the other compressionlib options +// for our database snapshots and is a widely adopted standard. +static const compression_algorithm SignalCompressionAlgorithm = COMPRESSION_LZMA; + +@implementation OWSBackupEncryptedItem + +@end + +#pragma mark - + +@interface OWSBackupIO () + +@property (nonatomic) NSString *jobTempDirPath; + +@end + +#pragma mark - + +@implementation OWSBackupIO + +- (instancetype)initWithJobTempDirPath:(NSString *)jobTempDirPath +{ + if (!(self = [super init])) { + return self; + } + + self.jobTempDirPath = jobTempDirPath; + + return self; +} + +#pragma mark - Encrypt + +- (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath +{ + OWSAssert(srcFilePath.length > 0); + + NSData *encryptionKey = [Randomness generateRandomBytes:(int)kOWSBackupKeyLength]; + + return [self encryptFileAsTempFile:srcFilePath encryptionKey:encryptionKey]; +} + +- (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath encryptionKey:(NSData *)encryptionKey +{ + OWSAssert(srcFilePath.length > 0); + OWSAssert(encryptionKey.length > 0); + + @autoreleasepool { + + // TODO: Encrypt the file without loading it into memory. + NSData *_Nullable srcData = [NSData dataWithContentsOfFile:srcFilePath]; + if (srcData.length < 1) { + OWSProdLogAndFail(@"%@ could not load file into memory for encryption.", self.logTag); + return nil; + } + return [self encryptDataAsTempFile:srcData encryptionKey:encryptionKey]; + } +} + +- (nullable OWSBackupEncryptedItem *)encryptDataAsTempFile:(NSData *)srcData +{ + OWSAssert(srcData); + + NSData *encryptionKey = [Randomness generateRandomBytes:(int)kOWSBackupKeyLength]; + + return [self encryptDataAsTempFile:srcData encryptionKey:encryptionKey]; +} + +- (nullable OWSBackupEncryptedItem *)encryptDataAsTempFile:(NSData *)unencryptedData + encryptionKey:(NSData *)encryptionKey +{ + OWSAssert(unencryptedData); + OWSAssert(encryptionKey.length > 0); + + @autoreleasepool { + + // TODO: Encrypt the data using key; + NSData *encryptedData = unencryptedData; + + NSString *dstFilePath = [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; + NSError *error; + BOOL success = [encryptedData writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; + if (!success || error) { + OWSProdLogAndFail(@"%@ error writing encrypted data: %@", self.logTag, error); + return nil; + } + [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; + OWSBackupEncryptedItem *item = [OWSBackupEncryptedItem new]; + item.filePath = dstFilePath; + item.encryptionKey = encryptionKey; + return item; + } +} + +#pragma mark - Decrypt + +- (BOOL)decryptFileAsFile:(NSString *)srcFilePath + dstFilePath:(NSString *)dstFilePath + encryptionKey:(NSData *)encryptionKey +{ + OWSAssert(srcFilePath.length > 0); + OWSAssert(encryptionKey.length > 0); + + @autoreleasepool { + + // TODO: Decrypt the file without loading it into memory. + NSData *data = [self decryptFileAsData:srcFilePath encryptionKey:encryptionKey]; + + if (data.length < 1) { + return NO; + } + + NSError *error; + BOOL success = [data writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; + if (!success || error) { + OWSProdLogAndFail(@"%@ error writing decrypted data: %@", self.logTag, error); + return NO; + } + [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; + + return YES; + } +} + +- (nullable NSData *)decryptFileAsData:(NSString *)srcFilePath encryptionKey:(NSData *)encryptionKey +{ + OWSAssert(srcFilePath.length > 0); + OWSAssert(encryptionKey.length > 0); + + @autoreleasepool { + + if (![NSFileManager.defaultManager fileExistsAtPath:srcFilePath]) { + DDLogError(@"%@ missing downloaded file.", self.logTag); + return nil; + } + + NSData *_Nullable srcData = [NSData dataWithContentsOfFile:srcFilePath]; + if (srcData.length < 1) { + OWSProdLogAndFail(@"%@ could not load file into memory for decryption.", self.logTag); + return nil; + } + + NSData *_Nullable dstData = [self decryptDataAsData:srcData encryptionKey:encryptionKey]; + return dstData; + } +} + +- (nullable NSData *)decryptDataAsData:(NSData *)encryptedData encryptionKey:(NSData *)encryptionKey +{ + OWSAssert(encryptedData); + OWSAssert(encryptionKey.length > 0); + + @autoreleasepool { + + // TODO: Decrypt the data using key; + NSData *unencryptedData = encryptedData; + + return unencryptedData; + } +} + +#pragma mark - Compression + +- (nullable NSData *)compressData:(NSData *)srcData +{ + OWSAssert(srcData); + + @autoreleasepool { + + if (!srcData) { + OWSProdLogAndFail(@"%@ missing unencrypted data.", self.logTag); + return nil; + } + + size_t srcLength = [srcData length]; + const uint8_t *srcBuffer = (const uint8_t *)[srcData bytes]; + if (!srcBuffer) { + return nil; + } + // This assumes that dst will always be smaller than src. + // + // We slightly pad the buffer size to account for the worst case. + size_t dstBufferLength = srcLength + 64 * 1024; + uint8_t *dstBuffer = malloc(sizeof(uint8_t) * dstBufferLength); + if (!dstBuffer) { + return nil; + } + size_t dstLength = compression_encode_buffer( + dstBuffer, dstBufferLength, srcBuffer, srcLength, NULL, SignalCompressionAlgorithm); + NSData *compressedData = [NSData dataWithBytesNoCopy:dstBuffer length:dstLength freeWhenDone:YES]; + + DDLogVerbose(@"%@ compressed %zd -> %zd = %0.2f", + self.logTag, + srcLength, + dstLength, + (srcLength > 0 ? (dstLength / (CGFloat)srcLength) : 0)); + + return compressedData; + } +} + +- (nullable NSData *)decompressData:(NSData *)srcData uncompressedDataLength:(NSUInteger)uncompressedDataLength +{ + OWSAssert(srcData); + + @autoreleasepool { + + if (!srcData) { + OWSProdLogAndFail(@"%@ missing unencrypted data.", self.logTag); + return nil; + } + + size_t srcLength = [srcData length]; + const uint8_t *srcBuffer = (const uint8_t *)[srcData bytes]; + if (!srcBuffer) { + return nil; + } + // We pad the buffer to be defensive. + size_t dstBufferLength = uncompressedDataLength + 1024; + uint8_t *dstBuffer = malloc(sizeof(uint8_t) * dstBufferLength); + if (!dstBuffer) { + return nil; + } + size_t dstLength = compression_decode_buffer( + dstBuffer, dstBufferLength, srcBuffer, srcLength, NULL, SignalCompressionAlgorithm); + NSData *decompressedData = [NSData dataWithBytesNoCopy:dstBuffer length:dstLength freeWhenDone:YES]; + OWSAssert(decompressedData.length == uncompressedDataLength); + DDLogVerbose(@"%@ decompressed %zd -> %zd = %0.2f", + self.logTag, + srcLength, + dstLength, + (dstLength > 0 ? (srcLength / (CGFloat)dstLength) : 0)); + + return decompressedData; + } +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackupImportJob.m b/Signal/src/util/OWSBackupImportJob.m index eae4ad2d90..76171cc3f8 100644 --- a/Signal/src/util/OWSBackupImportJob.m +++ b/Signal/src/util/OWSBackupImportJob.m @@ -3,12 +3,12 @@ // #import "OWSBackupImportJob.h" +#import "OWSBackupIO.h" #import "OWSDatabaseMigration.h" #import "OWSDatabaseMigrationRunner.h" #import "Signal-Swift.h" #import #import -#import #import #import #import @@ -20,20 +20,36 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe #pragma mark - +@interface OWSBackupImportItem : NSObject + +@property (nonatomic) NSString *recordName; + +@property (nonatomic) NSData *encryptionKey; + +@property (nonatomic, nullable) NSString *relativeFilePath; + +@property (nonatomic, nullable) NSString *downloadFilePath; + +@property (nonatomic, nullable) NSNumber *uncompressedDataLength; + +@end + +#pragma mark - + +@implementation OWSBackupImportItem + +@end + +#pragma mark - + @interface OWSBackupImportJob () @property (nonatomic, nullable) OWSBackgroundTask *backgroundTask; -@property (nonatomic, nullable) OWSBackupStorage *backupStorage; +@property (nonatomic) OWSBackupIO *backupIO; -// A map of "record name"-to-"file name". -@property (nonatomic) NSMutableDictionary *databaseRecordMap; - -// A map of "record name"-to-"file relative path". -@property (nonatomic) NSMutableDictionary *attachmentRecordMap; - -// A map of "record name"-to-"downloaded file path". -@property (nonatomic) NSMutableDictionary *downloadedFileMap; +@property (nonatomic) NSArray *databaseItems; +@property (nonatomic) NSArray *attachmentsItems; @end @@ -82,7 +98,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe progress:nil]; __weak OWSBackupImportJob *weakSelf = self; - [weakSelf downloadAndProcessManifest:^(NSError *_Nullable manifestError) { + [weakSelf downloadAndProcessManifestWithCompletion:^(NSError *_Nullable manifestError) { if (manifestError) { [weakSelf failWithError:manifestError]; return; @@ -92,13 +108,13 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return; } - NSMutableArray *allRecordNames = [NSMutableArray new]; - [allRecordNames addObjectsFromArray:weakSelf.databaseRecordMap.allKeys]; - // TODO: We could skip attachments that have already been restored - // by previous "backup import" attempts. - [allRecordNames addObjectsFromArray:weakSelf.attachmentRecordMap.allKeys]; + OWSAssert(self.databaseItems); + OWSAssert(self.attachmentsItems); + NSMutableArray *allItems = [NSMutableArray new]; + [allItems addObjectsFromArray:self.databaseItems]; + [allItems addObjectsFromArray:self.attachmentsItems]; [weakSelf - downloadFilesFromCloud:allRecordNames + downloadFilesFromCloud:allItems completion:^(NSError *_Nullable fileDownloadError) { if (fileDownloadError) { [weakSelf failWithError:fileDownloadError]; @@ -115,7 +131,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return; } - [weakSelf restoreDatabase:^(BOOL restoreDatabaseSuccess) { + [weakSelf restoreDatabaseWithCompletion:^(BOOL restoreDatabaseSuccess) { if (!restoreDatabaseSuccess) { [weakSelf failWithErrorDescription:NSLocalizedString( @"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT", @@ -128,7 +144,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return; } - [weakSelf ensureMigrations:^(BOOL ensureMigrationsSuccess) { + [weakSelf ensureMigrationsWithCompletion:^(BOOL ensureMigrationsSuccess) { if (!ensureMigrationsSuccess) { [weakSelf failWithErrorDescription:NSLocalizedString( @"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT", @@ -156,10 +172,13 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe OWSProdLogAndFail(@"%@ Could not create jobTempDirPath.", self.logTag); return NO; } + + self.backupIO = [[OWSBackupIO alloc] initWithJobTempDirPath:self.jobTempDirPath]; + return YES; } -- (void)downloadAndProcessManifest:(OWSBackupJobCompletion)completion +- (void)downloadAndProcessManifestWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); @@ -182,14 +201,15 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe } failure:^(NSError *error) { // The manifest file is critical so any error downloading it is unrecoverable. - OWSProdLogAndFail(@"%@ Could not download manifest.", self.logTag); + OWSProdLogAndFail(@"%@ Could not download manifest.", weakSelf.logTag); completion(error); }]; } -- (void)processManifest:(NSData *)manifestData completion:(OWSBackupJobBoolCompletion)completion +- (void)processManifest:(NSData *)manifestDataEncrypted completion:(OWSBackupJobBoolCompletion)completion { OWSAssert(completion); + OWSAssert(self.backupIO); if (self.isComplete) { return; @@ -197,9 +217,16 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + NSData *_Nullable manifestDataDecrypted = + [self.backupIO decryptDataAsData:manifestDataEncrypted encryptionKey:self.delegate.backupEncryptionKey]; + if (!manifestDataDecrypted) { + OWSProdLogAndFail(@"%@ Could not decrypt manifest.", self.logTag); + return completion(NO); + } + NSError *error; NSDictionary *_Nullable json = - [NSJSONSerialization JSONObjectWithData:manifestData options:0 error:&error]; + [NSJSONSerialization JSONObjectWithData:manifestDataDecrypted options:0 error:&error]; if (![json isKindOfClass:[NSDictionary class]]) { OWSProdLogAndFail(@"%@ Could not download manifest.", self.logTag); return completion(NO); @@ -207,49 +234,97 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe DDLogVerbose(@"%@ json: %@", self.logTag, json); - NSDictionary *_Nullable databaseRecordMap = json[kOWSBackup_ManifestKey_DatabaseFiles]; - NSDictionary *_Nullable attachmentRecordMap = json[kOWSBackup_ManifestKey_AttachmentFiles]; - NSString *_Nullable databaseKeySpecBase64 = json[kOWSBackup_ManifestKey_DatabaseKeySpec]; - if (!([databaseRecordMap isKindOfClass:[NSDictionary class]] && - [attachmentRecordMap isKindOfClass:[NSDictionary class]] && - [databaseKeySpecBase64 isKindOfClass:[NSString class]])) { - OWSProdLogAndFail(@"%@ Invalid manifest.", self.logTag); + NSArray *_Nullable databaseItems = + [self parseItems:json key:kOWSBackup_ManifestKey_DatabaseFiles]; + if (!databaseItems) { return completion(NO); } - NSData *_Nullable databaseKeySpec = [NSData dataFromBase64String:databaseKeySpecBase64]; - if (!databaseKeySpec) { - OWSProdLogAndFail(@"%@ Invalid manifest databaseKeySpec.", self.logTag); - return completion(NO); - } - if (![OWSBackupJob storeDatabaseKeySpec:databaseKeySpec keychainKey:kOWSBackup_ImportDatabaseKeySpec]) { - OWSProdLogAndFail(@"%@ Couldn't store databaseKeySpec from manifest.", self.logTag); + NSArray *_Nullable attachmentsItems = + [self parseItems:json key:kOWSBackup_ManifestKey_AttachmentFiles]; + if (!attachmentsItems) { return completion(NO); } - self.databaseRecordMap = [databaseRecordMap mutableCopy]; - self.attachmentRecordMap = [attachmentRecordMap mutableCopy]; + self.databaseItems = databaseItems; + self.attachmentsItems = attachmentsItems; return completion(YES); } -- (void)downloadFilesFromCloud:(NSMutableArray *)recordNames completion:(OWSBackupJobCompletion)completion +- (nullable NSArray *)parseItems:(id)json key:(NSString *)key { - OWSAssert(recordNames.count > 0); + OWSAssert(json); + OWSAssert(key.length); + + if (![json isKindOfClass:[NSDictionary class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key); + return nil; + } + NSArray *itemMaps = json[key]; + if (![itemMaps isKindOfClass:[NSArray class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key); + return nil; + } + NSMutableArray *items = [NSMutableArray new]; + for (NSDictionary *itemMap in itemMaps) { + if (![itemMap isKindOfClass:[NSDictionary class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid item: %@.", self.logTag, key); + return nil; + } + NSString *_Nullable recordName = itemMap[kOWSBackup_ManifestKey_RecordName]; + NSString *_Nullable encryptionKeyString = itemMap[kOWSBackup_ManifestKey_EncryptionKey]; + NSString *_Nullable relativeFilePath = itemMap[kOWSBackup_ManifestKey_RelativeFilePath]; + NSNumber *_Nullable uncompressedDataLength = itemMap[kOWSBackup_ManifestKey_DataSize]; + if (![recordName isKindOfClass:[NSString class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid recordName: %@.", self.logTag, key); + return nil; + } + if (![encryptionKeyString isKindOfClass:[NSString class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid encryptionKey: %@.", self.logTag, key); + return nil; + } + // relativeFilePath is an optional field. + if (relativeFilePath && ![relativeFilePath isKindOfClass:[NSString class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid relativeFilePath: %@.", self.logTag, key); + return nil; + } + NSData *_Nullable encryptionKey = [NSData dataFromBase64String:encryptionKeyString]; + if (!encryptionKey) { + OWSProdLogAndFail(@"%@ manifest has corrupt encryptionKey: %@.", self.logTag, key); + return nil; + } + // uncompressedDataLength is an optional field. + if (uncompressedDataLength && ![uncompressedDataLength isKindOfClass:[NSNumber class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid uncompressedDataLength: %@.", self.logTag, key); + return nil; + } + + OWSBackupImportItem *item = [OWSBackupImportItem new]; + item.recordName = recordName; + item.encryptionKey = encryptionKey; + item.relativeFilePath = relativeFilePath; + item.uncompressedDataLength = uncompressedDataLength; + [items addObject:item]; + } + return items; +} + +- (void)downloadFilesFromCloud:(NSMutableArray *)items + completion:(OWSBackupJobCompletion)completion +{ + OWSAssert(items.count > 0); OWSAssert(completion); DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - // A map of "record name"-to-"downloaded file path". - self.downloadedFileMap = [NSMutableDictionary new]; - - [self downloadNextFileFromCloud:recordNames recordCount:recordNames.count completion:completion]; + [self downloadNextItemFromCloud:items recordCount:items.count completion:completion]; } -- (void)downloadNextFileFromCloud:(NSMutableArray *)recordNames +- (void)downloadNextItemFromCloud:(NSMutableArray *)items recordCount:(NSUInteger)recordCount completion:(OWSBackupJobCompletion)completion { - OWSAssert(recordNames); + OWSAssert(items); OWSAssert(completion); if (self.isComplete) { @@ -257,43 +332,43 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return completion(nil); } - if (recordNames.count < 1) { + if (items.count < 1) { // All downloads are complete; exit. return completion(nil); } - NSString *recordName = recordNames.lastObject; - [recordNames removeLastObject]; + OWSBackupImportItem *item = items.lastObject; + [items removeLastObject]; + CGFloat progress = (recordCount > 0 ? ((recordCount - items.count) / (CGFloat)recordCount) : 0.f); [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_DOWNLOAD", @"Indicates that the backup import data is being downloaded.") - progress:@((recordCount - recordNames.count) / (CGFloat)recordCount)]; - - if (![recordName isKindOfClass:[NSString class]]) { - DDLogError(@"%@ invalid record name in manifest: %@", self.logTag, [recordName class]); - // Invalid record name in the manifest. This may be recoverable. - // Ignore this for now and proceed with the other downloads. - return [self downloadNextFileFromCloud:recordNames recordCount:recordCount completion:completion]; - } + progress:@(progress)]; // Use a predictable file path so that multiple "import backup" attempts // will leverage successful file downloads from previous attempts. - NSString *tempFilePath = [self.jobTempDirPath stringByAppendingPathComponent:recordName]; + // + // TODO: This will also require imports using a predictable jobTempDirPath. + NSString *tempFilePath = [self.jobTempDirPath stringByAppendingPathComponent:item.recordName]; // Skip redundant file download. if ([NSFileManager.defaultManager fileExistsAtPath:tempFilePath]) { [OWSFileSystem protectFileOrFolderAtPath:tempFilePath]; - self.downloadedFileMap[recordName] = tempFilePath; - [self downloadNextFileFromCloud:recordNames recordCount:recordCount completion:completion]; + + item.downloadFilePath = tempFilePath; + + [self downloadNextItemFromCloud:items recordCount:recordCount completion:completion]; return; } - [OWSBackupAPI downloadFileFromCloudWithRecordName:recordName + __weak OWSBackupImportJob *weakSelf = self; + [OWSBackupAPI downloadFileFromCloudWithRecordName:item.recordName toFileUrl:[NSURL fileURLWithPath:tempFilePath] success:^{ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ [OWSFileSystem protectFileOrFolderAtPath:tempFilePath]; - self.downloadedFileMap[recordName] = tempFilePath; - [self downloadNextFileFromCloud:recordNames recordCount:recordCount completion:completion]; + item.downloadFilePath = tempFilePath; + + [weakSelf downloadNextItemFromCloud:items recordCount:recordCount completion:completion]; }); } failure:^(NSError *error) { @@ -303,115 +378,66 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe - (void)restoreAttachmentFiles { - DDLogVerbose(@"%@ %s: %zd", self.logTag, __PRETTY_FUNCTION__, self.attachmentRecordMap.count); + DDLogVerbose(@"%@ %s: %zd", self.logTag, __PRETTY_FUNCTION__, self.attachmentsItems.count); NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder]; NSUInteger count = 0; - for (NSString *recordName in self.attachmentRecordMap) { + for (OWSBackupImportItem *item in self.attachmentsItems) { if (self.isComplete) { return; } + if (item.recordName.length < 1) { + DDLogError(@"%@ attachment was not downloaded.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + continue; + } + if (item.relativeFilePath.length < 1) { + DDLogError(@"%@ attachment missing relative file path.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + continue; + } count++; [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_FILES", @"Indicates that the backup import data is being restored.") - progress:@(count / (CGFloat)self.attachmentRecordMap.count)]; + progress:@(count / (CGFloat)self.attachmentsItems.count)]; - - NSString *dstRelativePath = self.attachmentRecordMap[recordName]; - if (! - [self restoreFileWithRecordName:recordName dstRelativePath:dstRelativePath dstDirPath:attachmentsDirPath]) { - // Attachment-related errors are recoverable and can be ignored. + NSString *dstFilePath = [attachmentsDirPath stringByAppendingPathComponent:item.relativeFilePath]; + if ([NSFileManager.defaultManager fileExistsAtPath:dstFilePath]) { + DDLogError(@"%@ skipping redundant file restore: %@.", self.logTag, dstFilePath); continue; } + @autoreleasepool { + if (![self.backupIO decryptFileAsFile:item.downloadFilePath + dstFilePath:dstFilePath + encryptionKey:item.encryptionKey]) { + DDLogError(@"%@ attachment could not be restored.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + continue; + } + } + + DDLogError(@"%@ restored file: %@.", self.logTag, item.relativeFilePath); } } -- (void)restoreDatabase:(OWSBackupJobBoolCompletion)completion +- (void)restoreDatabaseWithCompletion:(OWSBackupJobBoolCompletion)completion { OWSAssert(completion); DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_DATABASE", - @"Indicates that the backup database is being restored.") - progress:nil]; - - NSString *jobDatabaseDirPath = [self.jobTempDirPath stringByAppendingPathComponent:@"database"]; - if (![OWSFileSystem ensureDirectoryExists:jobDatabaseDirPath]) { - OWSProdLogAndFail(@"%@ Could not create jobDatabaseDirPath.", self.logTag); - return completion(NO); - } - - for (NSString *recordName in self.databaseRecordMap) { - if (self.isComplete) { - return completion(NO); - } - - NSString *dstRelativePath = self.databaseRecordMap[recordName]; - if (! - [self restoreFileWithRecordName:recordName dstRelativePath:dstRelativePath dstDirPath:jobDatabaseDirPath]) { - // Database-related errors are unrecoverable. - return completion(NO); - } - } - - BackupStorageKeySpecBlock keySpecBlock = ^{ - NSData *_Nullable databaseKeySpec = - [OWSBackupJob loadDatabaseKeySpecWithKeychainKey:kOWSBackup_ImportDatabaseKeySpec]; - if (!databaseKeySpec) { - OWSProdLogAndFail(@"%@ Could not load database keyspec for import.", self.logTag); - } - return databaseKeySpec; - }; - self.backupStorage = - [[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock]; - if (!self.backupStorage) { - OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag); - return completion(NO); - } - - // TODO: Do we really need to run these registrations on the main thread? - __weak OWSBackupImportJob *weakSelf = self; - dispatch_async(dispatch_get_main_queue(), ^{ - [weakSelf.backupStorage runSyncRegistrations]; - [weakSelf.backupStorage runAsyncRegistrationsWithCompletion:^{ - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - [weakSelf restoreDatabaseContents:completion]; - }); - }]; - }); -} - -- (void)restoreDatabaseContents:(OWSBackupJobBoolCompletion)completion -{ - OWSAssert(self.backupStorage); - OWSAssert(completion); - - DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - if (self.isComplete) { return completion(NO); } - YapDatabaseConnection *_Nullable tempDBConnection = self.backupStorage.newDatabaseConnection; - if (!tempDBConnection) { - OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag); - return completion(NO); - } - YapDatabaseConnection *_Nullable primaryDBConnection = self.primaryStorage.newDatabaseConnection; - if (!primaryDBConnection) { - OWSProdLogAndFail(@"%@ Could not create primaryDBConnection.", self.logTag); + YapDatabaseConnection *_Nullable dbConnection = self.primaryStorage.newDatabaseConnection; + if (!dbConnection) { + OWSProdLogAndFail(@"%@ Could not create dbConnection.", self.logTag); return completion(NO); } - NSDictionary *collectionTypeMap = @{ - [TSThread collection] : [TSThread class], - [TSAttachment collection] : [TSAttachment class], - [TSInteraction collection] : [TSInteraction class], - [OWSDatabaseMigration collection] : [OWSDatabaseMigration class], - }; // Order matters here. NSArray *collectionsToRestore = @[ [TSThread collection], @@ -424,90 +450,130 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe NSMutableDictionary *restoredEntityCounts = [NSMutableDictionary new]; __block unsigned long long copiedEntities = 0; __block BOOL aborted = NO; - [tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *srcTransaction) { - [primaryDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { - if (![srcTransaction boolForKey:kOWSBackup_Snapshot_ValidKey - inCollection:kOWSBackup_Snapshot_Collection - defaultValue:NO]) { - DDLogError(@"%@ invalid database.", self.logTag); + [dbConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { + for (NSString *collection in collectionsToRestore) { + if ([collection isEqualToString:[OWSDatabaseMigration collection]]) { + // It's okay if there are existing migrations; we'll clear those + // before restoring. + continue; + } + if ([transaction numberOfKeysInCollection:collection] > 0) { + DDLogError(@"%@ unexpected contents in database (%@).", self.logTag, collection); + } + } + + // Clear existing database contents. + // + // This should be safe since we only ever import into an empty database. + // + // Note that if the app receives a message after registering and before restoring + // backup, it will be lost. + // + // Note that this will clear all migrations. + for (NSString *collection in collectionsToRestore) { + [transaction removeAllObjectsInCollection:collection]; + } + + NSUInteger count = 0; + for (OWSBackupImportItem *item in self.databaseItems) { + if (self.isComplete) { + return; + } + if (item.recordName.length < 1) { + DDLogError(@"%@ database snapshot was not downloaded.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + if (!item.uncompressedDataLength || item.uncompressedDataLength.unsignedIntValue < 1) { + DDLogError(@"%@ database snapshot missing size.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + // Database-related errors are unrecoverable. aborted = YES; return completion(NO); } - for (NSString *collection in collectionsToRestore) { - if ([collection isEqualToString:[OWSDatabaseMigration collection]]) { - // It's okay if there are existing migrations; we'll clear those - // before restoring. - continue; + count++; + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_DATABASE", + @"Indicates that the backup database is being restored.") + progress:@(count / (CGFloat)self.databaseItems.count)]; + + @autoreleasepool { + NSData *_Nullable compressedData = + [self.backupIO decryptFileAsData:item.downloadFilePath encryptionKey:item.encryptionKey]; + if (!compressedData) { + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); } - if ([dstTransaction numberOfKeysInCollection:collection] > 0) { - DDLogError(@"%@ unexpected contents in database (%@).", self.logTag, collection); + NSData *_Nullable uncompressedData = + [self.backupIO decompressData:compressedData + uncompressedDataLength:item.uncompressedDataLength.unsignedIntValue]; + if (!uncompressedData) { + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + OWSSignaliOSProtosBackupSnapshot *_Nullable entities = + [OWSSignaliOSProtosBackupSnapshot parseFromData:uncompressedData]; + if (!entities || entities.entity.count < 1) { + DDLogError(@"%@ missing entities.", self.logTag); + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + for (OWSSignaliOSProtosBackupSnapshotBackupEntity *entity in entities.entity) { + NSData *_Nullable entityData = entity.entityData; + if (entityData.length < 1) { + DDLogError(@"%@ missing entity data.", self.logTag); + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + + __block TSYapDatabaseObject *object = nil; + @try { + NSKeyedUnarchiver *unarchiver = [[NSKeyedUnarchiver alloc] initForReadingWithData:entityData]; + object = [unarchiver decodeObjectForKey:@"root"]; + if (![object isKindOfClass:[object class]]) { + DDLogError(@"%@ invalid decoded entity: %@.", self.logTag, [object class]); + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + } @catch (NSException *exception) { + DDLogError(@"%@ could not decode entity.", self.logTag); + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + + [object saveWithTransaction:transaction]; + copiedEntities++; + NSString *collection = [object.class collection]; + NSUInteger restoredEntityCount = restoredEntityCounts[collection].unsignedIntValue; + restoredEntityCounts[collection] = @(restoredEntityCount + 1); } } - - // Clear existing database contents. - // - // This should be safe since we only ever import into an empty database. - // - // Note that if the app receives a message after registering and before restoring - // backup, it will be lost. - // - // Note that this will clear all migrations. - for (NSString *collection in collectionsToRestore) { - [dstTransaction removeAllObjectsInCollection:collection]; - } - - // Copy database entities. - for (NSString *collection in collectionsToRestore) { - [srcTransaction enumerateKeysAndObjectsInCollection:collection - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - aborted = YES; - return; - } - Class expectedType = collectionTypeMap[collection]; - OWSAssert(expectedType); - if (![object isKindOfClass:expectedType]) { - OWSProdLogAndFail(@"%@ unexpected class: %@ != %@", - self.logTag, - [object class], - expectedType); - return; - } - TSYapDatabaseObject *databaseObject = object; - [databaseObject saveWithTransaction:dstTransaction]; - - NSUInteger count - = restoredEntityCounts[collection].unsignedIntValue; - restoredEntityCounts[collection] = @(count + 1); - copiedEntities++; - }]; - } - }]; + } }]; - if (aborted) { + if (self.isComplete || aborted) { return; } - for (NSString *collection in collectionsToRestore) { - Class expectedType = collectionTypeMap[collection]; - OWSAssert(expectedType); - DDLogInfo(@"%@ copied %@ (%@): %@", self.logTag, expectedType, collection, restoredEntityCounts[collection]); + for (NSString *collection in restoredEntityCounts) { + DDLogInfo(@"%@ copied %@: %@", self.logTag, collection, restoredEntityCounts[collection]); } DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities); - [self.backupStorage logFileSizes]; - - // Close the database. - tempDBConnection = nil; - self.backupStorage = nil; + [self.primaryStorage logFileSizes]; completion(YES); } -- (void)ensureMigrations:(OWSBackupJobBoolCompletion)completion +- (void)ensureMigrationsWithCompletion:(OWSBackupJobBoolCompletion)completion { OWSAssert(completion); @@ -529,45 +595,6 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe }); } -- (BOOL)restoreFileWithRecordName:(NSString *)recordName - dstRelativePath:(NSString *)dstRelativePath - dstDirPath:(NSString *)dstDirPath -{ - OWSAssert(recordName); - OWSAssert(dstDirPath.length > 0); - - DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - - if (![recordName isKindOfClass:[NSString class]]) { - DDLogError(@"%@ invalid record name in manifest: %@", self.logTag, [recordName class]); - return NO; - } - if (![dstRelativePath isKindOfClass:[NSString class]]) { - DDLogError(@"%@ invalid dstRelativePath in manifest: %@", self.logTag, [recordName class]); - return NO; - } - NSString *dstFilePath = [dstDirPath stringByAppendingPathComponent:dstRelativePath]; - if ([NSFileManager.defaultManager fileExistsAtPath:dstFilePath]) { - DDLogError(@"%@ skipping redundant file restore: %@.", self.logTag, dstFilePath); - return YES; - } - NSString *downloadedFilePath = self.downloadedFileMap[recordName]; - if (![NSFileManager.defaultManager fileExistsAtPath:downloadedFilePath]) { - DDLogError(@"%@ missing downloaded attachment file.", self.logTag); - return NO; - } - NSError *error; - BOOL success = [NSFileManager.defaultManager moveItemAtPath:downloadedFilePath toPath:dstFilePath error:&error]; - if (!success || error) { - DDLogError(@"%@ could not restore attachment file.", self.logTag); - return NO; - } - - DDLogError(@"%@ restored file: %@ (%@).", self.logTag, dstFilePath, dstRelativePath); - - return YES; -} - @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackupJob.h b/Signal/src/util/OWSBackupJob.h index ee5b4aa0b8..4476cdcfe9 100644 --- a/Signal/src/util/OWSBackupJob.h +++ b/Signal/src/util/OWSBackupJob.h @@ -6,10 +6,10 @@ NS_ASSUME_NONNULL_BEGIN extern NSString *const kOWSBackup_ManifestKey_DatabaseFiles; extern NSString *const kOWSBackup_ManifestKey_AttachmentFiles; -extern NSString *const kOWSBackup_ManifestKey_DatabaseKeySpec; - -extern NSString *const kOWSBackup_Snapshot_Collection; -extern NSString *const kOWSBackup_Snapshot_ValidKey; +extern NSString *const kOWSBackup_ManifestKey_RecordName; +extern NSString *const kOWSBackup_ManifestKey_EncryptionKey; +extern NSString *const kOWSBackup_ManifestKey_RelativeFilePath; +extern NSString *const kOWSBackup_ManifestKey_DataSize; typedef void (^OWSBackupJobBoolCompletion)(BOOL success); typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error); @@ -18,8 +18,7 @@ typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error); @protocol OWSBackupJobDelegate -// TODO: Use actual key. -- (nullable NSData *)backupKey; +- (nullable NSData *)backupEncryptionKey; // Either backupJobDidSucceed:... or backupJobDidFail:... will // be called exactly once on the main thread UNLESS: @@ -64,23 +63,6 @@ typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error); - (void)failWithError:(NSError *)error; - (void)updateProgressWithDescription:(nullable NSString *)description progress:(nullable NSNumber *)progress; - -#pragma mark - Database KeySpec - -+ (nullable NSData *)loadDatabaseKeySpecWithKeychainKey:(NSString *)keychainKey; -+ (BOOL)storeDatabaseKeySpec:(NSData *)data keychainKey:(NSString *)keychainKey; -+ (BOOL)generateRandomDatabaseKeySpecWithKeychainKey:(NSString *)keychainKey; - -#pragma mark - Encryption - -+ (nullable NSString *)encryptFileAsTempFile:(NSString *)srcFilePath - jobTempDirPath:(NSString *)jobTempDirPath - delegate:(id)delegate; - -+ (nullable NSString *)encryptDataAsTempFile:(NSData *)data - jobTempDirPath:(NSString *)jobTempDirPath - delegate:(id)delegate; - @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackupJob.m b/Signal/src/util/OWSBackupJob.m index 84ba42b56c..04a35b4308 100644 --- a/Signal/src/util/OWSBackupJob.m +++ b/Signal/src/util/OWSBackupJob.m @@ -12,13 +12,13 @@ NS_ASSUME_NONNULL_BEGIN NSString *const kOWSBackup_ManifestKey_DatabaseFiles = @"database_files"; NSString *const kOWSBackup_ManifestKey_AttachmentFiles = @"attachment_files"; -NSString *const kOWSBackup_ManifestKey_DatabaseKeySpec = @"database_key_spec"; +NSString *const kOWSBackup_ManifestKey_RecordName = @"record_name"; +NSString *const kOWSBackup_ManifestKey_EncryptionKey = @"encryption_key"; +NSString *const kOWSBackup_ManifestKey_RelativeFilePath = @"relative_file_path"; +NSString *const kOWSBackup_ManifestKey_DataSize = @"data_size"; NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; -NSString *const kOWSBackup_Snapshot_Collection = @"kOWSBackup_Snapshot_Collection"; -NSString *const kOWSBackup_Snapshot_ValidKey = @"kOWSBackup_Snapshot_ValidKey"; - @interface OWSBackupJob () @property (nonatomic, weak) id delegate; @@ -146,100 +146,6 @@ NSString *const kOWSBackup_Snapshot_ValidKey = @"kOWSBackup_Snapshot_ValidKey"; }); } -#pragma mark - Database KeySpec - -+ (nullable NSData *)loadDatabaseKeySpecWithKeychainKey:(NSString *)keychainKey -{ - OWSAssert(keychainKey.length > 0); - - NSError *error; - NSData *_Nullable value = - [SAMKeychain passwordDataForService:kOWSBackup_KeychainService account:keychainKey error:&error]; - if (!value || error) { - DDLogError(@"%@ could not load database keyspec: %@", self.logTag, error); - } - return value; -} - -+ (BOOL)storeDatabaseKeySpec:(NSData *)data keychainKey:(NSString *)keychainKey -{ - OWSAssert(keychainKey.length > 0); - OWSAssert(data.length > 0); - - NSError *error; - [SAMKeychain setAccessibilityType:kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly]; - BOOL success = - [SAMKeychain setPasswordData:data forService:kOWSBackup_KeychainService account:keychainKey error:&error]; - if (!success || error) { - OWSFail(@"%@ Could not store database keyspec: %@.", self.logTag, error); - return NO; - } else { - return YES; - } -} - -+ (BOOL)generateRandomDatabaseKeySpecWithKeychainKey:(NSString *)keychainKey -{ - OWSAssert(keychainKey.length > 0); - - NSData *_Nullable databaseKeySpec = [Randomness generateRandomBytes:(int)kSQLCipherKeySpecLength]; - if (!databaseKeySpec) { - OWSFail(@"%@ Could not generate database keyspec.", self.logTag); - return NO; - } - - return [self storeDatabaseKeySpec:databaseKeySpec keychainKey:keychainKey]; -} - -#pragma mark - Encryption - -+ (nullable NSString *)encryptFileAsTempFile:(NSString *)srcFilePath - jobTempDirPath:(NSString *)jobTempDirPath - delegate:(id)delegate -{ - OWSAssert(srcFilePath.length > 0); - OWSAssert(jobTempDirPath.length > 0); - OWSAssert(delegate); - - // TODO: Encrypt the file using self.delegate.backupKey; - NSData *_Nullable backupKey = [delegate backupKey]; - OWSAssert(backupKey); - - NSString *dstFilePath = [jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; - NSFileManager *fileManager = [NSFileManager defaultManager]; - NSError *error; - BOOL success = [fileManager copyItemAtPath:srcFilePath toPath:dstFilePath error:&error]; - if (!success || error) { - OWSProdLogAndFail(@"%@ error writing encrypted file: %@", self.logTag, error); - return nil; - } - [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; - return dstFilePath; -} - -+ (nullable NSString *)encryptDataAsTempFile:(NSData *)data - jobTempDirPath:(NSString *)jobTempDirPath - delegate:(id)delegate -{ - OWSAssert(data); - OWSAssert(jobTempDirPath.length > 0); - OWSAssert(delegate); - - // TODO: Encrypt the file using self.delegate.backupKey; - NSData *_Nullable backupKey = [delegate backupKey]; - OWSAssert(backupKey); - - NSString *dstFilePath = [jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; - NSError *error; - BOOL success = [data writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; - if (!success || error) { - OWSProdLogAndFail(@"%@ error writing encrypted file: %@", self.logTag, error); - return nil; - } - [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; - return dstFilePath; -} - @end NS_ASSUME_NONNULL_END diff --git a/Signal/translations/en.lproj/Localizable.strings b/Signal/translations/en.lproj/Localizable.strings index e2e2de31c0..a20dbba54c 100644 --- a/Signal/translations/en.lproj/Localizable.strings +++ b/Signal/translations/en.lproj/Localizable.strings @@ -161,7 +161,7 @@ "BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT" = "Backup data could be exported."; /* Error indicating that the app received an invalid response from CloudKit. */ -"BACKUP_EXPORT_ERROR_INVALID_CLOUDKIT_RESPONSE" = "Invalid server response."; +"BACKUP_EXPORT_ERROR_INVALID_CLOUDKIT_RESPONSE" = "Invalid Service Response"; /* Error indicating the a backup export failed to save a file to the cloud. */ "BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED" = "Backup could not upload data."; @@ -172,6 +172,12 @@ /* Indicates that the backup export is being configured. */ "BACKUP_EXPORT_PHASE_CONFIGURATION" = "Initializing Backup"; +/* Indicates that the database data is being exported. */ +"BACKUP_EXPORT_PHASE_DATABASE_EXPORT" = "Exporting Data"; + +/* Indicates that the backup export data is being finalized. */ +"BACKUP_EXPORT_PHASE_DATABASE_FINALIZED" = "Finalizing Data"; + /* Indicates that the backup export data is being exported. */ "BACKUP_EXPORT_PHASE_EXPORT" = "Exporting Backup"; diff --git a/SignalServiceKit/protobuf/OWSSignaliOSProtos.proto b/SignalServiceKit/protobuf/OWSSignaliOSProtos.proto new file mode 100644 index 0000000000..901b598dc9 --- /dev/null +++ b/SignalServiceKit/protobuf/OWSSignaliOSProtos.proto @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2014-2016 Open Whisper Systems + * + * Licensed according to the LICENSE file in this repository. + */ + +package signalios; + +// Signal-iOS +import "objectivec-descriptor.proto"; +option (google.protobuf.objectivec_file_options).class_prefix = "OWSSignaliOSProtos"; + +message BackupSnapshot +{ + message BackupEntity + { + enum Type { + UNKNOWN = 0; + MIGRATION = 1; + THREAD = 2; + INTERACTION = 3; + ATTACHMENT = 4; + } + optional Type type = 1; + optional bytes entityData = 2; + } + + repeated BackupEntity entity = 1; +} \ No newline at end of file diff --git a/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.h b/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.h new file mode 100644 index 0000000000..603d5e390f --- /dev/null +++ b/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.h @@ -0,0 +1,186 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! + +#import + +// @@protoc_insertion_point(imports) + +@class OWSSignaliOSProtosBackupSnapshot; +@class OWSSignaliOSProtosBackupSnapshotBackupEntity; +@class OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder; +@class OWSSignaliOSProtosBackupSnapshotBuilder; +@class ObjectiveCFileOptions; +@class ObjectiveCFileOptionsBuilder; +@class PBDescriptorProto; +@class PBDescriptorProtoBuilder; +@class PBDescriptorProtoExtensionRange; +@class PBDescriptorProtoExtensionRangeBuilder; +@class PBEnumDescriptorProto; +@class PBEnumDescriptorProtoBuilder; +@class PBEnumOptions; +@class PBEnumOptionsBuilder; +@class PBEnumValueDescriptorProto; +@class PBEnumValueDescriptorProtoBuilder; +@class PBEnumValueOptions; +@class PBEnumValueOptionsBuilder; +@class PBFieldDescriptorProto; +@class PBFieldDescriptorProtoBuilder; +@class PBFieldOptions; +@class PBFieldOptionsBuilder; +@class PBFileDescriptorProto; +@class PBFileDescriptorProtoBuilder; +@class PBFileDescriptorSet; +@class PBFileDescriptorSetBuilder; +@class PBFileOptions; +@class PBFileOptionsBuilder; +@class PBMessageOptions; +@class PBMessageOptionsBuilder; +@class PBMethodDescriptorProto; +@class PBMethodDescriptorProtoBuilder; +@class PBMethodOptions; +@class PBMethodOptionsBuilder; +@class PBOneofDescriptorProto; +@class PBOneofDescriptorProtoBuilder; +@class PBServiceDescriptorProto; +@class PBServiceDescriptorProtoBuilder; +@class PBServiceOptions; +@class PBServiceOptionsBuilder; +@class PBSourceCodeInfo; +@class PBSourceCodeInfoBuilder; +@class PBSourceCodeInfoLocation; +@class PBSourceCodeInfoLocationBuilder; +@class PBUninterpretedOption; +@class PBUninterpretedOptionBuilder; +@class PBUninterpretedOptionNamePart; +@class PBUninterpretedOptionNamePartBuilder; + + +typedef NS_ENUM(SInt32, OWSSignaliOSProtosBackupSnapshotBackupEntityType) { + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown = 0, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeMigration = 1, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeThread = 2, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeInteraction = 3, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeAttachment = 4, +}; + +BOOL OWSSignaliOSProtosBackupSnapshotBackupEntityTypeIsValidValue(OWSSignaliOSProtosBackupSnapshotBackupEntityType value); +NSString *NSStringFromOWSSignaliOSProtosBackupSnapshotBackupEntityType(OWSSignaliOSProtosBackupSnapshotBackupEntityType value); + + +@interface OWSSignaliOSProtosOwssignaliOsprotosRoot : NSObject { +} ++ (PBExtensionRegistry*) extensionRegistry; ++ (void) registerAllExtensions:(PBMutableExtensionRegistry*) registry; +@end + +#define BackupSnapshot_entity @"entity" +@interface OWSSignaliOSProtosBackupSnapshot : PBGeneratedMessage { +@private + NSMutableArray * entityArray; +} +@property (readonly, strong) NSArray * entity; +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index; + ++ (instancetype) defaultInstance; +- (instancetype) defaultInstance; + +- (BOOL) isInitialized; +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output; +- (OWSSignaliOSProtosBackupSnapshotBuilder*) builder; ++ (OWSSignaliOSProtosBackupSnapshotBuilder*) builder; ++ (OWSSignaliOSProtosBackupSnapshotBuilder*) builderWithPrototype:(OWSSignaliOSProtosBackupSnapshot*) prototype; +- (OWSSignaliOSProtosBackupSnapshotBuilder*) toBuilder; + ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromData:(NSData*) data; ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input; ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input; ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; +@end + +#define BackupEntity_type @"type" +#define BackupEntity_entityData @"entityData" +@interface OWSSignaliOSProtosBackupSnapshotBackupEntity : PBGeneratedMessage { +@private + BOOL hasEntityData_:1; + BOOL hasType_:1; + NSData* entityData; + OWSSignaliOSProtosBackupSnapshotBackupEntityType type; +} +- (BOOL) hasType; +- (BOOL) hasEntityData; +@property (readonly) OWSSignaliOSProtosBackupSnapshotBackupEntityType type; +@property (readonly, strong) NSData* entityData; + ++ (instancetype) defaultInstance; +- (instancetype) defaultInstance; + +- (BOOL) isInitialized; +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builder; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builder; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builderWithPrototype:(OWSSignaliOSProtosBackupSnapshotBackupEntity*) prototype; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) toBuilder; + ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; +@end + +@interface OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder : PBGeneratedMessageBuilder { +@private + OWSSignaliOSProtosBackupSnapshotBackupEntity* resultBackupEntity; +} + +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) defaultInstance; + +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clear; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clone; + +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) build; +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) buildPartial; + +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFrom:(OWSSignaliOSProtosBackupSnapshotBackupEntity*) other; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; + +- (BOOL) hasType; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityType) type; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) setType:(OWSSignaliOSProtosBackupSnapshotBackupEntityType) value; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clearType; + +- (BOOL) hasEntityData; +- (NSData*) entityData; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) setEntityData:(NSData*) value; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clearEntityData; +@end + +@interface OWSSignaliOSProtosBackupSnapshotBuilder : PBGeneratedMessageBuilder { +@private + OWSSignaliOSProtosBackupSnapshot* resultBackupSnapshot; +} + +- (OWSSignaliOSProtosBackupSnapshot*) defaultInstance; + +- (OWSSignaliOSProtosBackupSnapshotBuilder*) clear; +- (OWSSignaliOSProtosBackupSnapshotBuilder*) clone; + +- (OWSSignaliOSProtosBackupSnapshot*) build; +- (OWSSignaliOSProtosBackupSnapshot*) buildPartial; + +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFrom:(OWSSignaliOSProtosBackupSnapshot*) other; +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input; +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; + +- (NSMutableArray *)entity; +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index; +- (OWSSignaliOSProtosBackupSnapshotBuilder *)addEntity:(OWSSignaliOSProtosBackupSnapshotBackupEntity*)value; +- (OWSSignaliOSProtosBackupSnapshotBuilder *)setEntityArray:(NSArray *)array; +- (OWSSignaliOSProtosBackupSnapshotBuilder *)clearEntity; +@end + + +// @@protoc_insertion_point(global_scope) diff --git a/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.m b/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.m new file mode 100644 index 0000000000..559fc36c05 --- /dev/null +++ b/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.m @@ -0,0 +1,535 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! + +#import "OWSSignaliOSProtos.pb.h" +// @@protoc_insertion_point(imports) + +@implementation OWSSignaliOSProtosOwssignaliOsprotosRoot +static PBExtensionRegistry* extensionRegistry = nil; ++ (PBExtensionRegistry*) extensionRegistry { + return extensionRegistry; +} + ++ (void) initialize { + if (self == [OWSSignaliOSProtosOwssignaliOsprotosRoot class]) { + PBMutableExtensionRegistry* registry = [PBMutableExtensionRegistry registry]; + [self registerAllExtensions:registry]; + [ObjectivecDescriptorRoot registerAllExtensions:registry]; + extensionRegistry = registry; + } +} ++ (void) registerAllExtensions:(PBMutableExtensionRegistry*) registry { +} +@end + +@interface OWSSignaliOSProtosBackupSnapshot () +@property (strong) NSMutableArray * entityArray; +@end + +@implementation OWSSignaliOSProtosBackupSnapshot + +@synthesize entityArray; +@dynamic entity; +- (instancetype) init { + if ((self = [super init])) { + } + return self; +} +static OWSSignaliOSProtosBackupSnapshot* defaultOWSSignaliOSProtosBackupSnapshotInstance = nil; ++ (void) initialize { + if (self == [OWSSignaliOSProtosBackupSnapshot class]) { + defaultOWSSignaliOSProtosBackupSnapshotInstance = [[OWSSignaliOSProtosBackupSnapshot alloc] init]; + } +} ++ (instancetype) defaultInstance { + return defaultOWSSignaliOSProtosBackupSnapshotInstance; +} +- (instancetype) defaultInstance { + return defaultOWSSignaliOSProtosBackupSnapshotInstance; +} +- (NSArray *)entity { + return entityArray; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index { + return [entityArray objectAtIndex:index]; +} +- (BOOL) isInitialized { + return YES; +} +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output { + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignaliOSProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + [output writeMessage:1 value:element]; + }]; + [self.unknownFields writeToCodedOutputStream:output]; +} +- (SInt32) serializedSize { + __block SInt32 size_ = memoizedSerializedSize; + if (size_ != -1) { + return size_; + } + + size_ = 0; + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignaliOSProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + size_ += computeMessageSize(1, element); + }]; + size_ += self.unknownFields.serializedSize; + memoizedSerializedSize = size_; + return size_; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromData:(NSData*) data { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromData:data] build]; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromData:data extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromInputStream:input] build]; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromCodedInputStream:input] build]; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromCodedInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBuilder*) builder { + return [[OWSSignaliOSProtosBackupSnapshotBuilder alloc] init]; +} ++ (OWSSignaliOSProtosBackupSnapshotBuilder*) builderWithPrototype:(OWSSignaliOSProtosBackupSnapshot*) prototype { + return [[OWSSignaliOSProtosBackupSnapshot builder] mergeFrom:prototype]; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) builder { + return [OWSSignaliOSProtosBackupSnapshot builder]; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) toBuilder { + return [OWSSignaliOSProtosBackupSnapshot builderWithPrototype:self]; +} +- (void) writeDescriptionTo:(NSMutableString*) output withIndent:(NSString*) indent { + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignaliOSProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + [output appendFormat:@"%@%@ {\n", indent, @"entity"]; + [element writeDescriptionTo:output + withIndent:[NSString stringWithFormat:@"%@ ", indent]]; + [output appendFormat:@"%@}\n", indent]; + }]; + [self.unknownFields writeDescriptionTo:output withIndent:indent]; +} +- (void) storeInDictionary:(NSMutableDictionary *)dictionary { + for (OWSSignaliOSProtosBackupSnapshotBackupEntity* element in self.entityArray) { + NSMutableDictionary *elementDictionary = [NSMutableDictionary dictionary]; + [element storeInDictionary:elementDictionary]; + [dictionary setObject:[NSDictionary dictionaryWithDictionary:elementDictionary] forKey:@"entity"]; + } + [self.unknownFields storeInDictionary:dictionary]; +} +- (BOOL) isEqual:(id)other { + if (other == self) { + return YES; + } + if (![other isKindOfClass:[OWSSignaliOSProtosBackupSnapshot class]]) { + return NO; + } + OWSSignaliOSProtosBackupSnapshot *otherMessage = other; + return + [self.entityArray isEqualToArray:otherMessage.entityArray] && + (self.unknownFields == otherMessage.unknownFields || (self.unknownFields != nil && [self.unknownFields isEqual:otherMessage.unknownFields])); +} +- (NSUInteger) hash { + __block NSUInteger hashCode = 7; + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignaliOSProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + hashCode = hashCode * 31 + [element hash]; + }]; + hashCode = hashCode * 31 + [self.unknownFields hash]; + return hashCode; +} +@end + +@interface OWSSignaliOSProtosBackupSnapshotBackupEntity () +@property OWSSignaliOSProtosBackupSnapshotBackupEntityType type; +@property (strong) NSData* entityData; +@end + +@implementation OWSSignaliOSProtosBackupSnapshotBackupEntity + +- (BOOL) hasType { + return !!hasType_; +} +- (void) setHasType:(BOOL) _value_ { + hasType_ = !!_value_; +} +@synthesize type; +- (BOOL) hasEntityData { + return !!hasEntityData_; +} +- (void) setHasEntityData:(BOOL) _value_ { + hasEntityData_ = !!_value_; +} +@synthesize entityData; +- (instancetype) init { + if ((self = [super init])) { + self.type = OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown; + self.entityData = [NSData data]; + } + return self; +} +static OWSSignaliOSProtosBackupSnapshotBackupEntity* defaultOWSSignaliOSProtosBackupSnapshotBackupEntityInstance = nil; ++ (void) initialize { + if (self == [OWSSignaliOSProtosBackupSnapshotBackupEntity class]) { + defaultOWSSignaliOSProtosBackupSnapshotBackupEntityInstance = [[OWSSignaliOSProtosBackupSnapshotBackupEntity alloc] init]; + } +} ++ (instancetype) defaultInstance { + return defaultOWSSignaliOSProtosBackupSnapshotBackupEntityInstance; +} +- (instancetype) defaultInstance { + return defaultOWSSignaliOSProtosBackupSnapshotBackupEntityInstance; +} +- (BOOL) isInitialized { + return YES; +} +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output { + if (self.hasType) { + [output writeEnum:1 value:self.type]; + } + if (self.hasEntityData) { + [output writeData:2 value:self.entityData]; + } + [self.unknownFields writeToCodedOutputStream:output]; +} +- (SInt32) serializedSize { + __block SInt32 size_ = memoizedSerializedSize; + if (size_ != -1) { + return size_; + } + + size_ = 0; + if (self.hasType) { + size_ += computeEnumSize(1, self.type); + } + if (self.hasEntityData) { + size_ += computeDataSize(2, self.entityData); + } + size_ += self.unknownFields.serializedSize; + memoizedSerializedSize = size_; + return size_; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromData:data] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromData:data extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromInputStream:input] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromCodedInputStream:input] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromCodedInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builder { + return [[OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder alloc] init]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builderWithPrototype:(OWSSignaliOSProtosBackupSnapshotBackupEntity*) prototype { + return [[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFrom:prototype]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builder { + return [OWSSignaliOSProtosBackupSnapshotBackupEntity builder]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) toBuilder { + return [OWSSignaliOSProtosBackupSnapshotBackupEntity builderWithPrototype:self]; +} +- (void) writeDescriptionTo:(NSMutableString*) output withIndent:(NSString*) indent { + if (self.hasType) { + [output appendFormat:@"%@%@: %@\n", indent, @"type", NSStringFromOWSSignaliOSProtosBackupSnapshotBackupEntityType(self.type)]; + } + if (self.hasEntityData) { + [output appendFormat:@"%@%@: %@\n", indent, @"entityData", self.entityData]; + } + [self.unknownFields writeDescriptionTo:output withIndent:indent]; +} +- (void) storeInDictionary:(NSMutableDictionary *)dictionary { + if (self.hasType) { + [dictionary setObject: @(self.type) forKey: @"type"]; + } + if (self.hasEntityData) { + [dictionary setObject: self.entityData forKey: @"entityData"]; + } + [self.unknownFields storeInDictionary:dictionary]; +} +- (BOOL) isEqual:(id)other { + if (other == self) { + return YES; + } + if (![other isKindOfClass:[OWSSignaliOSProtosBackupSnapshotBackupEntity class]]) { + return NO; + } + OWSSignaliOSProtosBackupSnapshotBackupEntity *otherMessage = other; + return + self.hasType == otherMessage.hasType && + (!self.hasType || self.type == otherMessage.type) && + self.hasEntityData == otherMessage.hasEntityData && + (!self.hasEntityData || [self.entityData isEqual:otherMessage.entityData]) && + (self.unknownFields == otherMessage.unknownFields || (self.unknownFields != nil && [self.unknownFields isEqual:otherMessage.unknownFields])); +} +- (NSUInteger) hash { + __block NSUInteger hashCode = 7; + if (self.hasType) { + hashCode = hashCode * 31 + self.type; + } + if (self.hasEntityData) { + hashCode = hashCode * 31 + [self.entityData hash]; + } + hashCode = hashCode * 31 + [self.unknownFields hash]; + return hashCode; +} +@end + +BOOL OWSSignaliOSProtosBackupSnapshotBackupEntityTypeIsValidValue(OWSSignaliOSProtosBackupSnapshotBackupEntityType value) { + switch (value) { + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown: + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeMigration: + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeThread: + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeInteraction: + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeAttachment: + return YES; + default: + return NO; + } +} +NSString *NSStringFromOWSSignaliOSProtosBackupSnapshotBackupEntityType(OWSSignaliOSProtosBackupSnapshotBackupEntityType value) { + switch (value) { + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown: + return @"OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown"; + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeMigration: + return @"OWSSignaliOSProtosBackupSnapshotBackupEntityTypeMigration"; + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeThread: + return @"OWSSignaliOSProtosBackupSnapshotBackupEntityTypeThread"; + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeInteraction: + return @"OWSSignaliOSProtosBackupSnapshotBackupEntityTypeInteraction"; + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeAttachment: + return @"OWSSignaliOSProtosBackupSnapshotBackupEntityTypeAttachment"; + default: + return nil; + } +} + +@interface OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder() +@property (strong) OWSSignaliOSProtosBackupSnapshotBackupEntity* resultBackupEntity; +@end + +@implementation OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder +@synthesize resultBackupEntity; +- (instancetype) init { + if ((self = [super init])) { + self.resultBackupEntity = [[OWSSignaliOSProtosBackupSnapshotBackupEntity alloc] init]; + } + return self; +} +- (PBGeneratedMessage*) internalGetResult { + return resultBackupEntity; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clear { + self.resultBackupEntity = [[OWSSignaliOSProtosBackupSnapshotBackupEntity alloc] init]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clone { + return [OWSSignaliOSProtosBackupSnapshotBackupEntity builderWithPrototype:resultBackupEntity]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) defaultInstance { + return [OWSSignaliOSProtosBackupSnapshotBackupEntity defaultInstance]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) build { + [self checkInitialized]; + return [self buildPartial]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) buildPartial { + OWSSignaliOSProtosBackupSnapshotBackupEntity* returnMe = resultBackupEntity; + self.resultBackupEntity = nil; + return returnMe; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFrom:(OWSSignaliOSProtosBackupSnapshotBackupEntity*) other { + if (other == [OWSSignaliOSProtosBackupSnapshotBackupEntity defaultInstance]) { + return self; + } + if (other.hasType) { + [self setType:other.type]; + } + if (other.hasEntityData) { + [self setEntityData:other.entityData]; + } + [self mergeUnknownFields:other.unknownFields]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input { + return [self mergeFromCodedInputStream:input extensionRegistry:[PBExtensionRegistry emptyRegistry]]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + PBUnknownFieldSetBuilder* unknownFields = [PBUnknownFieldSet builderWithUnknownFields:self.unknownFields]; + while (YES) { + SInt32 tag = [input readTag]; + switch (tag) { + case 0: + [self setUnknownFields:[unknownFields build]]; + return self; + default: { + if (![self parseUnknownField:input unknownFields:unknownFields extensionRegistry:extensionRegistry tag:tag]) { + [self setUnknownFields:[unknownFields build]]; + return self; + } + break; + } + case 8: { + OWSSignaliOSProtosBackupSnapshotBackupEntityType value = (OWSSignaliOSProtosBackupSnapshotBackupEntityType)[input readEnum]; + if (OWSSignaliOSProtosBackupSnapshotBackupEntityTypeIsValidValue(value)) { + [self setType:value]; + } else { + [unknownFields mergeVarintField:1 value:value]; + } + break; + } + case 18: { + [self setEntityData:[input readData]]; + break; + } + } + } +} +- (BOOL) hasType { + return resultBackupEntity.hasType; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityType) type { + return resultBackupEntity.type; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) setType:(OWSSignaliOSProtosBackupSnapshotBackupEntityType) value { + resultBackupEntity.hasType = YES; + resultBackupEntity.type = value; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clearType { + resultBackupEntity.hasType = NO; + resultBackupEntity.type = OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown; + return self; +} +- (BOOL) hasEntityData { + return resultBackupEntity.hasEntityData; +} +- (NSData*) entityData { + return resultBackupEntity.entityData; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) setEntityData:(NSData*) value { + resultBackupEntity.hasEntityData = YES; + resultBackupEntity.entityData = value; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clearEntityData { + resultBackupEntity.hasEntityData = NO; + resultBackupEntity.entityData = [NSData data]; + return self; +} +@end + +@interface OWSSignaliOSProtosBackupSnapshotBuilder() +@property (strong) OWSSignaliOSProtosBackupSnapshot* resultBackupSnapshot; +@end + +@implementation OWSSignaliOSProtosBackupSnapshotBuilder +@synthesize resultBackupSnapshot; +- (instancetype) init { + if ((self = [super init])) { + self.resultBackupSnapshot = [[OWSSignaliOSProtosBackupSnapshot alloc] init]; + } + return self; +} +- (PBGeneratedMessage*) internalGetResult { + return resultBackupSnapshot; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) clear { + self.resultBackupSnapshot = [[OWSSignaliOSProtosBackupSnapshot alloc] init]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) clone { + return [OWSSignaliOSProtosBackupSnapshot builderWithPrototype:resultBackupSnapshot]; +} +- (OWSSignaliOSProtosBackupSnapshot*) defaultInstance { + return [OWSSignaliOSProtosBackupSnapshot defaultInstance]; +} +- (OWSSignaliOSProtosBackupSnapshot*) build { + [self checkInitialized]; + return [self buildPartial]; +} +- (OWSSignaliOSProtosBackupSnapshot*) buildPartial { + OWSSignaliOSProtosBackupSnapshot* returnMe = resultBackupSnapshot; + self.resultBackupSnapshot = nil; + return returnMe; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFrom:(OWSSignaliOSProtosBackupSnapshot*) other { + if (other == [OWSSignaliOSProtosBackupSnapshot defaultInstance]) { + return self; + } + if (other.entityArray.count > 0) { + if (resultBackupSnapshot.entityArray == nil) { + resultBackupSnapshot.entityArray = [[NSMutableArray alloc] initWithArray:other.entityArray]; + } else { + [resultBackupSnapshot.entityArray addObjectsFromArray:other.entityArray]; + } + } + [self mergeUnknownFields:other.unknownFields]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input { + return [self mergeFromCodedInputStream:input extensionRegistry:[PBExtensionRegistry emptyRegistry]]; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + PBUnknownFieldSetBuilder* unknownFields = [PBUnknownFieldSet builderWithUnknownFields:self.unknownFields]; + while (YES) { + SInt32 tag = [input readTag]; + switch (tag) { + case 0: + [self setUnknownFields:[unknownFields build]]; + return self; + default: { + if (![self parseUnknownField:input unknownFields:unknownFields extensionRegistry:extensionRegistry tag:tag]) { + [self setUnknownFields:[unknownFields build]]; + return self; + } + break; + } + case 10: { + OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder* subBuilder = [OWSSignaliOSProtosBackupSnapshotBackupEntity builder]; + [input readMessage:subBuilder extensionRegistry:extensionRegistry]; + [self addEntity:[subBuilder buildPartial]]; + break; + } + } + } +} +- (NSMutableArray *)entity { + return resultBackupSnapshot.entityArray; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index { + return [resultBackupSnapshot entityAtIndex:index]; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder *)addEntity:(OWSSignaliOSProtosBackupSnapshotBackupEntity*)value { + if (resultBackupSnapshot.entityArray == nil) { + resultBackupSnapshot.entityArray = [[NSMutableArray alloc]init]; + } + [resultBackupSnapshot.entityArray addObject:value]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder *)setEntityArray:(NSArray *)array { + resultBackupSnapshot.entityArray = [[NSMutableArray alloc]initWithArray:array]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder *)clearEntity { + resultBackupSnapshot.entityArray = nil; + return self; +} +@end + + +// @@protoc_insertion_point(global_scope) diff --git a/SignalServiceKit/src/Storage/OWSBackupStorage.h b/SignalServiceKit/src/Storage/OWSBackupStorage.h deleted file mode 100644 index babb381714..0000000000 --- a/SignalServiceKit/src/Storage/OWSBackupStorage.h +++ /dev/null @@ -1,30 +0,0 @@ -// -// Copyright (c) 2018 Open Whisper Systems. All rights reserved. -// - -// -// Copyright (c) 2018 Open Whisper Systems. All rights reserved. -// -#import "OWSPrimaryStorage.h" -#import "OWSStorage.h" - -NS_ASSUME_NONNULL_BEGIN - -typedef NSData *_Nullable (^BackupStorageKeySpecBlock)(void); - -@interface OWSBackupStorage : OWSStorage - -- (instancetype)init NS_UNAVAILABLE; - -- (instancetype)initStorage NS_UNAVAILABLE; - -- (instancetype)initBackupStorageWithDatabaseDirPath:(NSString *)databaseDirPath - keySpecBlock:(BackupStorageKeySpecBlock)keySpecBlock NS_DESIGNATED_INITIALIZER; - -- (void)runSyncRegistrations; -- (void)runAsyncRegistrationsWithCompletion:(void (^_Nonnull)(void))completion; -- (BOOL)areAllRegistrationsComplete; - -@end - -NS_ASSUME_NONNULL_END diff --git a/SignalServiceKit/src/Storage/OWSBackupStorage.m b/SignalServiceKit/src/Storage/OWSBackupStorage.m deleted file mode 100644 index 38200e543c..0000000000 --- a/SignalServiceKit/src/Storage/OWSBackupStorage.m +++ /dev/null @@ -1,153 +0,0 @@ -// -// Copyright (c) 2018 Open Whisper Systems. All rights reserved. -// - -#import "OWSBackupStorage.h" -#import "OWSFileSystem.h" -#import "OWSStorage+Subclass.h" - -NS_ASSUME_NONNULL_BEGIN - -@interface OWSBackupStorage () - -@property (atomic) BOOL areAsyncRegistrationsComplete; -@property (atomic) BOOL areSyncRegistrationsComplete; - -@property (nonatomic, readonly) NSString *databaseDirPath; -@property (nonatomic, readonly) BackupStorageKeySpecBlock keySpecBlock; - -@end - -#pragma mark - - -@implementation OWSBackupStorage - -- (instancetype)initBackupStorageWithDatabaseDirPath:(NSString *)databaseDirPath - keySpecBlock:(BackupStorageKeySpecBlock)keySpecBlock -{ - OWSAssert(databaseDirPath.length > 0); - OWSAssert(keySpecBlock); - OWSAssert([OWSFileSystem ensureDirectoryExists:databaseDirPath]); - - self = [super initStorage]; - - if (self) { - _databaseDirPath = databaseDirPath; - _keySpecBlock = keySpecBlock; - - [self loadDatabase]; - } - - return self; -} - -- (void)loadDatabase -{ - [super loadDatabase]; - - [self protectFiles]; -} - -- (void)resetStorage -{ - [super resetStorage]; -} - -- (void)runSyncRegistrations -{ - runSyncRegistrationsForStorage(self); - - // See comments on OWSDatabaseConnection. - // - // In the absence of finding documentation that can shed light on the issue we've been - // seeing, this issue only seems to affect sync and not async registrations. We've always - // been opening write transactions before the async registrations complete without negative - // consequences. - OWSAssert(!self.areSyncRegistrationsComplete); - self.areSyncRegistrationsComplete = YES; -} - -- (void)runAsyncRegistrationsWithCompletion:(void (^_Nonnull)(void))completion -{ - OWSAssert(completion); - - runAsyncRegistrationsForStorage(self); - - DDLogVerbose(@"%@ async registrations enqueued.", self.logTag); - - // Block until all async registrations are complete. - // - // NOTE: This has to happen on the "registration connection" for this - // database. - YapDatabaseConnection *dbConnection = self.registrationConnection; - OWSAssert(self.registrationConnection); - [dbConnection flushTransactionsWithCompletionQueue:dispatch_get_main_queue() - completionBlock:^{ - OWSAssert(!self.areAsyncRegistrationsComplete); - - DDLogVerbose(@"%@ async registrations complete.", self.logTag); - - self.areAsyncRegistrationsComplete = YES; - - completion(); - }]; -} - -- (void)protectFiles -{ - [self logFileSizes]; - - // Protect the entire new database directory. - [OWSFileSystem protectFileOrFolderAtPath:self.databaseDirPath]; -} - -+ (NSString *)databaseFilename -{ - return @"SignalBackup.sqlite"; -} - -- (NSString *)databaseFilename -{ - return OWSBackupStorage.databaseFilename; -} - -- (NSString *)databaseFilename_SHM -{ - return [self.databaseFilename stringByAppendingString:@"-shm"]; -} - -- (NSString *)databaseFilename_WAL -{ - return [self.databaseFilename stringByAppendingString:@"-wal"]; -} - -- (NSString *)databaseFilePath -{ - return [self.databaseDirPath stringByAppendingPathComponent:self.databaseFilename]; -} - -- (NSString *)databaseFilePath_SHM -{ - return [self.databaseDirPath stringByAppendingPathComponent:self.databaseFilename_SHM]; -} - -- (NSString *)databaseFilePath_WAL -{ - return [self.databaseDirPath stringByAppendingPathComponent:self.databaseFilename_WAL]; -} - -- (NSData *)databaseKeySpec -{ - OWSAssert(self.keySpecBlock); - - return self.keySpecBlock(); -} - -- (void)ensureDatabaseKeySpecExists -{ - // Do nothing. -} - -@end - -NS_ASSUME_NONNULL_END diff --git a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m index 3d575e4f7d..e059475679 100644 --- a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m +++ b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m @@ -326,6 +326,16 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage) return OWSPrimaryStorage.databaseFilePath; } +- (NSString *)databaseFilePath_SHM +{ + return OWSPrimaryStorage.databaseFilePath_SHM; +} + +- (NSString *)databaseFilePath_WAL +{ + return OWSPrimaryStorage.databaseFilePath_WAL; +} + - (NSString *)databaseFilename_SHM { return OWSPrimaryStorage.databaseFilename_SHM; diff --git a/SignalServiceKit/src/Storage/OWSStorage+Subclass.h b/SignalServiceKit/src/Storage/OWSStorage+Subclass.h index 1417b4fb20..4a4232c290 100644 --- a/SignalServiceKit/src/Storage/OWSStorage+Subclass.h +++ b/SignalServiceKit/src/Storage/OWSStorage+Subclass.h @@ -6,8 +6,12 @@ NS_ASSUME_NONNULL_BEGIN +@class YapDatabase; + @interface OWSStorage (Subclass) +@property (atomic, nullable, readonly) YapDatabase *database; + - (void)loadDatabase; - (void)runSyncRegistrations; diff --git a/SignalServiceKit/src/Storage/OWSStorage.h b/SignalServiceKit/src/Storage/OWSStorage.h index 4e0857cb51..75f710db16 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.h +++ b/SignalServiceKit/src/Storage/OWSStorage.h @@ -61,6 +61,8 @@ extern NSString *const StorageIsReadyNotification; - (nullable id)registeredExtension:(NSString *)extensionName; - (unsigned long long)databaseFileSize; +- (unsigned long long)databaseWALFileSize; +- (unsigned long long)databaseSHMFileSize; - (YapDatabaseConnection *)registrationConnection; diff --git a/SignalServiceKit/src/Storage/OWSStorage.m b/SignalServiceKit/src/Storage/OWSStorage.m index 0b84a0e780..73fc3680db 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.m +++ b/SignalServiceKit/src/Storage/OWSStorage.m @@ -58,7 +58,7 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); OWSAssert(delegate); - _delegate = delegate; + self.delegate = delegate; return self; } @@ -134,9 +134,8 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); @property (atomic, weak) id delegate; -@property (atomic, nullable) YapDatabaseConnection *registrationConnectionCached; - - (instancetype)init NS_UNAVAILABLE; + - (id)initWithPath:(NSString *)inPath serializer:(nullable YapDatabaseSerializer)inSerializer deserializer:(YapDatabaseDeserializer)inDeserializer @@ -163,7 +162,7 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); OWSAssert(delegate); - _delegate = delegate; + self.delegate = delegate; return self; } @@ -184,21 +183,15 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); - (YapDatabaseConnection *)registrationConnection { - @synchronized(self) - { - if (!self.registrationConnectionCached) { - YapDatabaseConnection *connection = [super registrationConnection]; + YapDatabaseConnection *connection = [super registrationConnection]; #ifdef DEBUG - // Flag the registration connection as such. - OWSAssert([connection isKindOfClass:[OWSDatabaseConnection class]]); - ((OWSDatabaseConnection *)connection).canWriteBeforeStorageReady = YES; + // Flag the registration connection as such. + OWSAssert([connection isKindOfClass:[OWSDatabaseConnection class]]); + ((OWSDatabaseConnection *)connection).canWriteBeforeStorageReady = YES; #endif - self.registrationConnectionCached = connection; - } - return self.registrationConnectionCached; - } + return connection; } @end @@ -278,6 +271,9 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); - (void)dealloc { + // Surface memory leaks by logging the deallocation of this class. + DDLogVerbose(@"Dealloc: %@", self.class); + [[NSNotificationCenter defaultCenter] removeObserver:self]; } @@ -363,11 +359,6 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); for (OWSStorage *storage in self.allStorages) { [storage runAsyncRegistrationsWithCompletion:^{ if ([self postRegistrationCompleteNotificationIfPossible]) { - // If all registrations are complete, clean up the - // registration process. - - ((OWSDatabase *)storage.database).registrationConnectionCached = nil; - backgroundTask = nil; } }]; @@ -408,15 +399,23 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); - (BOOL)tryToLoadDatabase { + __weak OWSStorage *weakSelf = self; + YapDatabaseOptions *options = [[YapDatabaseOptions alloc] init]; options.corruptAction = YapDatabaseCorruptAction_Fail; options.enableMultiProcessSupport = YES; options.cipherKeySpecBlock = ^{ + // NOTE: It's critical that we don't capture a reference to self + // (e.g. by using OWSAssert()) or this database will contain a + // circular reference and will leak. + OWSStorage *strongSelf = weakSelf; + OWSCAssert(strongSelf); + // Rather than compute this once and capture the value of the key // in the closure, we prefer to fetch the key from the keychain multiple times // in order to keep the key out of application memory. - NSData *databaseKeySpec = [self databaseKeySpec]; - OWSAssert(databaseKeySpec.length == kSQLCipherKeySpecLength); + NSData *databaseKeySpec = [strongSelf databaseKeySpec]; + OWSCAssert(databaseKeySpec.length == kSQLCipherKeySpecLength); return databaseKeySpec; }; @@ -709,6 +708,16 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); return [OWSFileSystem fileSizeOfPath:self.databaseFilePath].unsignedLongLongValue; } +- (unsigned long long)databaseWALFileSize +{ + return [OWSFileSystem fileSizeOfPath:self.databaseFilePath_WAL].unsignedLongLongValue; +} + +- (unsigned long long)databaseSHMFileSize +{ + return [OWSFileSystem fileSizeOfPath:self.databaseFilePath_SHM].unsignedLongLongValue; +} + + (nullable NSData *)tryToLoadKeyChainValue:(NSString *)keychainKey errorHandle:(NSError **)errorHandle { OWSAssert(keychainKey.length > 0); diff --git a/SignalServiceKit/src/Util/OWSFileSystem.h b/SignalServiceKit/src/Util/OWSFileSystem.h index 70ec5492bf..e9a1802898 100644 --- a/SignalServiceKit/src/Util/OWSFileSystem.h +++ b/SignalServiceKit/src/Util/OWSFileSystem.h @@ -28,9 +28,9 @@ NS_ASSUME_NONNULL_BEGIN // Returns NO IFF the directory does not exist and could not be created. + (BOOL)ensureDirectoryExists:(NSString *)dirPath; -+ (void)deleteFile:(NSString *)filePath; ++ (BOOL)deleteFile:(NSString *)filePath; -+ (void)deleteFileIfExists:(NSString *)filePath; ++ (BOOL)deleteFileIfExists:(NSString *)filePath; + (NSArray *_Nullable)allFilesInDirectoryRecursive:(NSString *)dirPath error:(NSError **)error; diff --git a/SignalServiceKit/src/Util/OWSFileSystem.m b/SignalServiceKit/src/Util/OWSFileSystem.m index ddcb724e9d..fa83b41a39 100644 --- a/SignalServiceKit/src/Util/OWSFileSystem.m +++ b/SignalServiceKit/src/Util/OWSFileSystem.m @@ -227,20 +227,23 @@ NS_ASSUME_NONNULL_BEGIN } } -+ (void)deleteFile:(NSString *)filePath ++ (BOOL)deleteFile:(NSString *)filePath { NSError *error; - [[NSFileManager defaultManager] removeItemAtPath:filePath error:&error]; - if (error) { + BOOL success = [[NSFileManager defaultManager] removeItemAtPath:filePath error:&error]; + if (!success || error) { DDLogError(@"%@ Failed to delete file: %@", self.logTag, error.description); + return NO; } + return YES; } -+ (void)deleteFileIfExists:(NSString *)filePath ++ (BOOL)deleteFileIfExists:(NSString *)filePath { - if ([[NSFileManager defaultManager] fileExistsAtPath:filePath]) { - [self deleteFile:filePath]; + if (![[NSFileManager defaultManager] fileExistsAtPath:filePath]) { + return YES; } + return [self deleteFile:filePath]; } + (NSArray *_Nullable)allFilesInDirectoryRecursive:(NSString *)dirPath error:(NSError **)error