From 1bbd41f725ee44fb69d6d472d9da3b21376a959f Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Fri, 16 Mar 2018 08:48:21 -0300 Subject: [PATCH 01/13] Improve perf of database snapshots. --- .../AppSettings/AboutTableViewController.m | 18 +- .../AppSettings/AppSettingsViewController.m | 5 + .../ViewControllers/DebugUI/DebugUIBackup.m | 56 ++- .../src/ViewControllers/HomeViewController.m | 4 + Signal/src/network/GiphyAPI.swift | 79 ++-- Signal/src/util/OWSBackupExportJob.m | 382 +++++++++++++++++- .../translations/en.lproj/Localizable.strings | 8 +- .../src/Storage/OWSPrimaryStorage.m | 10 + .../src/Storage/OWSStorage+Subclass.h | 4 + SignalServiceKit/src/Storage/OWSStorage.h | 2 + SignalServiceKit/src/Storage/OWSStorage.m | 61 ++- 11 files changed, 536 insertions(+), 93 deletions(-) diff --git a/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m b/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m index 4652b6294a..babc5d5adb 100644 --- a/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m +++ b/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m @@ -72,18 +72,28 @@ #ifdef DEBUG __block NSUInteger threadCount; __block NSUInteger messageCount; + __block NSUInteger attachmentCount; [OWSPrimaryStorage.dbReadConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) { - threadCount = [[transaction ext:TSThreadDatabaseViewExtensionName] numberOfItemsInAllGroups]; - messageCount = [[transaction ext:TSMessageDatabaseViewExtensionName] numberOfItemsInAllGroups]; + threadCount = [transaction numberOfKeysInCollection:[TSThread collection]]; + messageCount = [transaction numberOfKeysInCollection:[TSInteraction collection]]; + attachmentCount = [transaction numberOfKeysInCollection:[TSAttachment collection]]; }]; - unsigned long long databaseFileSize = [OWSPrimaryStorage.sharedManager databaseFileSize]; OWSTableSection *debugSection = [OWSTableSection new]; debugSection.headerTitle = @"Debug"; [debugSection addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Threads: %zd", threadCount]]]; [debugSection addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Messages: %zd", messageCount]]]; [debugSection - addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database size: %llu", databaseFileSize]]]; + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Attachments: %zd", attachmentCount]]]; + [debugSection + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database size: %llu", + [OWSPrimaryStorage.sharedManager databaseFileSize]]]]; + [debugSection + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database WAL size: %llu", + [OWSPrimaryStorage.sharedManager databaseWALFileSize]]]]; + [debugSection + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database SHM size: %llu", + [OWSPrimaryStorage.sharedManager databaseSHMFileSize]]]]; [contents addSection:debugSection]; OWSPreferences *preferences = [Environment preferences]; diff --git a/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m b/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m index 26f94e5e97..43e23fec77 100644 --- a/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m +++ b/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m @@ -90,6 +90,11 @@ self.title = NSLocalizedString(@"SETTINGS_NAV_BAR_TITLE", @"Title for settings activity"); [self updateTableContents]; + + dispatch_async(dispatch_get_main_queue(), ^{ + [self showBackup]; + // [self showDebugUI]; + }); } - (void)viewWillAppear:(BOOL)animated diff --git a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m index ee9d39b33a..8f46dcbaff 100644 --- a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m +++ b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m @@ -40,6 +40,10 @@ NS_ASSUME_NONNULL_BEGIN actionBlock:^{ [DebugUIBackup tryToImportBackup]; }]]; + [items addObject:[OWSTableItem itemWithTitle:@"Log Database Size Stats" + actionBlock:^{ + [DebugUIBackup logDatabaseSizeStats]; + }]]; return [OWSTableSection sectionWithTitle:self.name items:items]; } @@ -95,17 +99,57 @@ NS_ASSUME_NONNULL_BEGIN message:@"This will delete all of your database contents." preferredStyle:UIAlertControllerStyleAlert]; - [controller addAction:[UIAlertAction - actionWithTitle:@"Restore" - style:UIAlertActionStyleDefault - handler:^(UIAlertAction *_Nonnull action) { - [OWSBackup.sharedManager tryToImportBackup]; - }]]; + [controller addAction:[UIAlertAction actionWithTitle:@"Restore" + style:UIAlertActionStyleDefault + handler:^(UIAlertAction *_Nonnull action) { + [OWSBackup.sharedManager tryToImportBackup]; + }]]; [controller addAction:[OWSAlerts cancelAction]]; UIViewController *fromViewController = [[UIApplication sharedApplication] frontmostViewController]; [fromViewController presentViewController:controller animated:YES completion:nil]; } ++ (void)logDatabaseSizeStats +{ + DDLogInfo(@"%@ logDatabaseSizeStats.", self.logTag); + + __block unsigned long long interactionCount = 0; + __block unsigned long long interactionSizeTotal = 0; + __block unsigned long long attachmentCount = 0; + __block unsigned long long attachmentSizeTotal = 0; + [[OWSPrimaryStorage.sharedManager newDatabaseConnection] readWithBlock:^(YapDatabaseReadTransaction *transaction) { + [transaction enumerateKeysAndObjectsInCollection:[TSInteraction collection] + usingBlock:^(NSString *key, id object, BOOL *stop) { + TSInteraction *interaction = object; + interactionCount++; + NSData *_Nullable data = + [NSKeyedArchiver archivedDataWithRootObject:interaction]; + OWSAssert(data); + interactionSizeTotal += data.length; + }]; + [transaction enumerateKeysAndObjectsInCollection:[TSAttachment collection] + usingBlock:^(NSString *key, id object, BOOL *stop) { + TSAttachment *attachment = object; + attachmentCount++; + NSData *_Nullable data = + [NSKeyedArchiver archivedDataWithRootObject:attachment]; + OWSAssert(data); + attachmentSizeTotal += data.length; + }]; + }]; + + DDLogInfo(@"%@ interactionCount: %llu", self.logTag, interactionCount); + DDLogInfo(@"%@ interactionSizeTotal: %llu", self.logTag, interactionSizeTotal); + if (interactionCount > 0) { + DDLogInfo(@"%@ interaction average size: %f", self.logTag, interactionSizeTotal / (double)interactionCount); + } + DDLogInfo(@"%@ attachmentCount: %llu", self.logTag, attachmentCount); + DDLogInfo(@"%@ attachmentSizeTotal: %llu", self.logTag, attachmentSizeTotal); + if (attachmentCount > 0) { + DDLogInfo(@"%@ attachment average size: %f", self.logTag, attachmentSizeTotal / (double)attachmentCount); + } +} + @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/ViewControllers/HomeViewController.m b/Signal/src/ViewControllers/HomeViewController.m index dccd63e158..8d47def8b3 100644 --- a/Signal/src/ViewControllers/HomeViewController.m +++ b/Signal/src/ViewControllers/HomeViewController.m @@ -426,6 +426,10 @@ typedef NS_ENUM(NSInteger, CellState) { kArchiveState, kInboxState }; } [self checkIfEmptyView]; + + dispatch_async(dispatch_get_main_queue(), ^{ + [self settingsButtonPressed:nil]; + }); } - (void)viewWillDisappear:(BOOL)animated diff --git a/Signal/src/network/GiphyAPI.swift b/Signal/src/network/GiphyAPI.swift index 2ee16daa48..6bc738dfdf 100644 --- a/Signal/src/network/GiphyAPI.swift +++ b/Signal/src/network/GiphyAPI.swift @@ -1,5 +1,5 @@ // -// Copyright (c) 2017 Open Whisper Systems. All rights reserved. +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. // import Foundation @@ -122,33 +122,33 @@ extension GiphyError: LocalizedError { public func pickStillRendition() -> GiphyRendition? { // Stills are just temporary placeholders, so use the smallest still possible. - return pickRendition(renditionType: .stillPreview, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize) + return pickRendition(renditionType: .stillPreview, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize) } public func pickPreviewRendition() -> GiphyRendition? { // Try to pick a small file... - if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.largerIsBetter, maxFileSize:kPreferedPreviewFileSize) { + if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .largerIsBetter, maxFileSize: kPreferedPreviewFileSize) { return rendition } // ...but gradually relax the file restriction... - if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize * 2) { + if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize * 2) { return rendition } // ...and relax even more until we find an animated rendition. - return pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize * 3) + return pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize * 3) } public func pickSendingRendition() -> GiphyRendition? { // Try to pick a small file... - if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.largerIsBetter, maxFileSize:kPreferedSendingFileSize) { + if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .largerIsBetter, maxFileSize: kPreferedSendingFileSize) { return rendition } // ...but gradually relax the file restriction... - if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedSendingFileSize * 2) { + if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedSendingFileSize * 2) { return rendition } // ...and relax even more until we find an animated rendition. - return pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedSendingFileSize * 3) + return pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedSendingFileSize * 3) } enum RenditionType { @@ -299,12 +299,12 @@ extension GiphyError: LocalizedError { } private func giphyAPISessionManager() -> AFHTTPSessionManager? { - guard let baseUrl = NSURL(string:kGiphyBaseURL) else { + guard let baseUrl = NSURL(string: kGiphyBaseURL) else { Logger.error("\(TAG) Invalid base URL.") return nil } - let sessionManager = AFHTTPSessionManager(baseURL:baseUrl as URL, - sessionConfiguration:GiphyAPI.giphySessionConfiguration()) + let sessionManager = AFHTTPSessionManager(baseURL: baseUrl as URL, + sessionConfiguration: GiphyAPI.giphySessionConfiguration()) sessionManager.requestSerializer = AFJSONRequestSerializer() sessionManager.responseSerializer = AFJSONResponseSerializer() @@ -319,7 +319,7 @@ extension GiphyError: LocalizedError { failure(nil) return } - guard NSURL(string:kGiphyBaseURL) != nil else { + guard NSURL(string: kGiphyBaseURL) != nil else { Logger.error("\(TAG) Invalid base URL.") failure(nil) return @@ -338,10 +338,11 @@ extension GiphyError: LocalizedError { sessionManager.get(urlString, parameters: {}, - progress:nil, - success: { _, value in + progress: nil, + success: { task, value in Logger.error("\(GiphyAPI.TAG) search request succeeded") - guard let imageInfos = self.parseGiphyImages(responseJson:value) else { + Logger.error("\(GiphyAPI.TAG) search request succeeded \(task.originalRequest?.url)") + guard let imageInfos = self.parseGiphyImages(responseJson: value) else { failure(nil) return } @@ -355,16 +356,16 @@ extension GiphyError: LocalizedError { // MARK: Parse API Responses - private func parseGiphyImages(responseJson:Any?) -> [GiphyImageInfo]? { + private func parseGiphyImages(responseJson: Any?) -> [GiphyImageInfo]? { guard let responseJson = responseJson else { Logger.error("\(TAG) Missing response.") return nil } - guard let responseDict = responseJson as? [String:Any] else { + guard let responseDict = responseJson as? [String: Any] else { Logger.error("\(TAG) Invalid response.") return nil } - guard let imageDicts = responseDict["data"] as? [[String:Any]] else { + guard let imageDicts = responseDict["data"] as? [[String: Any]] else { Logger.error("\(TAG) Invalid response data.") return nil } @@ -374,7 +375,7 @@ extension GiphyError: LocalizedError { } // Giphy API results are often incomplete or malformed, so we need to be defensive. - private func parseGiphyImage(imageDict: [String:Any]) -> GiphyImageInfo? { + private func parseGiphyImage(imageDict: [String: Any]) -> GiphyImageInfo? { guard let giphyId = imageDict["id"] as? String else { Logger.warn("\(TAG) Image dict missing id.") return nil @@ -383,18 +384,18 @@ extension GiphyError: LocalizedError { Logger.warn("\(TAG) Image dict has invalid id.") return nil } - guard let renditionDicts = imageDict["images"] as? [String:Any] else { + guard let renditionDicts = imageDict["images"] as? [String: Any] else { Logger.warn("\(TAG) Image dict missing renditions.") return nil } var renditions = [GiphyRendition]() for (renditionName, renditionDict) in renditionDicts { - guard let renditionDict = renditionDict as? [String:Any] else { + guard let renditionDict = renditionDict as? [String: Any] else { Logger.warn("\(TAG) Invalid rendition dict.") continue } - guard let rendition = parseGiphyRendition(renditionName:renditionName, - renditionDict:renditionDict) else { + guard let rendition = parseGiphyRendition(renditionName: renditionName, + renditionDict: renditionDict) else { continue } renditions.append(rendition) @@ -404,13 +405,13 @@ extension GiphyError: LocalizedError { return nil } - guard let originalRendition = findOriginalRendition(renditions:renditions) else { + guard let originalRendition = findOriginalRendition(renditions: renditions) else { Logger.warn("\(TAG) Image has no original rendition.") return nil } - return GiphyImageInfo(giphyId : giphyId, - renditions : renditions, + return GiphyImageInfo(giphyId: giphyId, + renditions: renditions, originalRendition: originalRendition) } @@ -425,15 +426,15 @@ extension GiphyError: LocalizedError { // // We should discard renditions which are missing or have invalid properties. private func parseGiphyRendition(renditionName: String, - renditionDict: [String:Any]) -> GiphyRendition? { - guard let width = parsePositiveUInt(dict:renditionDict, key:"width", typeName:"rendition") else { + renditionDict: [String: Any]) -> GiphyRendition? { + guard let width = parsePositiveUInt(dict: renditionDict, key: "width", typeName: "rendition") else { return nil } - guard let height = parsePositiveUInt(dict:renditionDict, key:"height", typeName:"rendition") else { + guard let height = parsePositiveUInt(dict: renditionDict, key: "height", typeName: "rendition") else { return nil } // Be lenient when parsing file sizes - we don't require them for stills. - let fileSize = parseLenientUInt(dict:renditionDict, key:"size") + let fileSize = parseLenientUInt(dict: renditionDict, key: "size") guard let urlString = renditionDict["url"] as? String else { return nil } @@ -441,7 +442,7 @@ extension GiphyError: LocalizedError { Logger.warn("\(TAG) Rendition has invalid url.") return nil } - guard let url = NSURL(string:urlString) else { + guard let url = NSURL(string: urlString) else { Logger.warn("\(TAG) Rendition url could not be parsed.") return nil } @@ -464,16 +465,16 @@ extension GiphyError: LocalizedError { } return GiphyRendition( - format : format, - name : renditionName, - width : width, - height : height, - fileSize : fileSize, - url : url + format: format, + name: renditionName, + width: width, + height: height, + fileSize: fileSize, + url: url ) } - private func parsePositiveUInt(dict: [String:Any], key: String, typeName: String) -> UInt? { + private func parsePositiveUInt(dict: [String: Any], key: String, typeName: String) -> UInt? { guard let value = dict[key] else { return nil } @@ -490,7 +491,7 @@ extension GiphyError: LocalizedError { return parsedValue } - private func parseLenientUInt(dict: [String:Any], key: String) -> UInt { + private func parseLenientUInt(dict: [String: Any], key: String) -> UInt { let defaultValue = UInt(0) guard let value = dict[key] else { diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index 3a831c4cb5..8570da789a 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -5,6 +5,8 @@ #import "OWSBackupExportJob.h" #import "OWSDatabaseMigration.h" #import "Signal-Swift.h" +#import "zlib.h" +#import #import #import #import @@ -18,11 +20,167 @@ #import #import #import +#import NS_ASSUME_NONNULL_BEGIN NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKeySpec"; +@interface YapDatabase (OWSBackupExportJob) + +- (void)flushInternalQueue; +- (void)flushCheckpointQueue; + +@end + +#pragma mark - + +@interface OWSStorageReference : NSObject + +@property (nonatomic, nullable) OWSStorage *storage; + +@end + +#pragma mark - + +@implementation OWSStorageReference + +@end + +#pragma mark - + +// TODO: This implementation is a proof-of-concept and +// isn't production ready. +@interface OWSExportStream : NSObject + +@property (nonatomic) NSString *dataFilePath; + +@property (nonatomic) NSString *zipFilePath; + +@property (nonatomic, nullable) NSFileHandle *fileHandle; + +@property (nonatomic, nullable) SSZipArchive *zipFile; + +@end + +#pragma mark - + +@implementation OWSExportStream + +- (void)dealloc +{ + // Surface memory leaks by logging the deallocation of view controllers. + DDLogVerbose(@"Dealloc: %@", self.class); + + [self.fileHandle closeFile]; + + if (self.zipFile) { + if (![self.zipFile close]) { + DDLogError(@"%@ couldn't close to database snapshot zip.", self.logTag); + } + } +} + ++ (OWSExportStream *)exportStreamWithName:(NSString *)filename jobTempDirPath:(NSString *)jobTempDirPath +{ + OWSAssert(filename.length > 0); + OWSAssert(jobTempDirPath.length > 0); + + OWSExportStream *exportStream = [OWSExportStream new]; + exportStream.dataFilePath = [jobTempDirPath stringByAppendingPathComponent:filename]; + exportStream.zipFilePath = [exportStream.dataFilePath stringByAppendingPathExtension:@"zip"]; + if (![exportStream open]) { + return nil; + } + return exportStream; +} + +- (BOOL)open +{ + if (![[NSFileManager defaultManager] createFileAtPath:self.dataFilePath contents:nil attributes:nil]) { + OWSProdLogAndFail(@"%@ Could not create database snapshot stream.", self.logTag); + return NO; + } + if (![OWSFileSystem protectFileOrFolderAtPath:self.dataFilePath]) { + OWSProdLogAndFail(@"%@ Could not protect database snapshot stream.", self.logTag); + return NO; + } + NSError *error; + self.fileHandle = [NSFileHandle fileHandleForWritingToURL:[NSURL fileURLWithPath:self.dataFilePath] error:&error]; + if (!self.fileHandle || error) { + OWSProdLogAndFail(@"%@ Could not open database snapshot stream: %@.", self.logTag, error); + return NO; + } + return YES; +} + +- (BOOL)writeObject:(TSYapDatabaseObject *)object +{ + OWSAssert(object); + OWSAssert(self.fileHandle); + + NSData *_Nullable data = [NSKeyedArchiver archivedDataWithRootObject:object]; + if (!data) { + OWSProdLogAndFail(@"%@ couldn't serialize database object: %@", self.logTag, [object class]); + return NO; + } + + // We use a fixed width data type. + unsigned int dataLength = (unsigned int)data.length; + NSData *dataLengthData = [NSData dataWithBytes:&dataLength length:sizeof(dataLength)]; + [self.fileHandle writeData:dataLengthData]; + [self.fileHandle writeData:data]; + return YES; +} + +- (BOOL)closeAndZipData +{ + [self.fileHandle closeFile]; + self.fileHandle = nil; + + self.zipFile = [[SSZipArchive alloc] initWithPath:self.zipFilePath]; + if (!self.zipFile) { + OWSProdLogAndFail(@"%@ Could not create database snapshot zip.", self.logTag); + return NO; + } + if (![self.zipFile open]) { + OWSProdLogAndFail(@"%@ Could not open database snapshot zip.", self.logTag); + return NO; + } + + BOOL success = [self.zipFile writeFileAtPath:self.dataFilePath + withFileName:@"payload" + compressionLevel:Z_BEST_COMPRESSION + password:nil + AES:NO]; + if (!success) { + OWSProdLogAndFail(@"%@ Could not write to database snapshot zip.", self.logTag); + return NO; + } + + if (![self.zipFile close]) { + DDLogError(@"%@ couldn't close database snapshot zip.", self.logTag); + return NO; + } + self.zipFile = nil; + + if (![OWSFileSystem protectFileOrFolderAtPath:self.zipFilePath]) { + DDLogError(@"%@ could not protect database snapshot zip.", self.logTag); + } + + DDLogInfo(@"%@ wrote database snapshot zip: %@ (%@ -> %@)", + self.logTag, + self.zipFilePath.lastPathComponent, + [OWSFileSystem fileSizeOfPath:self.dataFilePath], + [OWSFileSystem fileSizeOfPath:self.zipFilePath]); + + return YES; +} + +@end + +#pragma mark - + @interface OWSAttachmentExport : NSObject @property (nonatomic, weak) id delegate; @@ -88,8 +246,6 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe @property (nonatomic, nullable) OWSBackgroundTask *backgroundTask; -@property (nonatomic, nullable) OWSBackupStorage *backupStorage; - @property (nonatomic) NSMutableArray *databaseFilePaths; // A map of "record name"-to-"file name". @property (nonatomic) NSMutableDictionary *databaseRecordMap; @@ -238,9 +394,10 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe return databaseKeySpec; }; - self.backupStorage = + OWSStorageReference *storageReference = [OWSStorageReference new]; + storageReference.storage = [[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock]; - if (!self.backupStorage) { + if (!storageReference.storage) { OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag); return completion(NO); } @@ -248,30 +405,144 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe // TODO: Do we really need to run these registrations on the main thread? __weak OWSBackupExportJob *weakSelf = self; dispatch_async(dispatch_get_main_queue(), ^{ - [weakSelf.backupStorage runSyncRegistrations]; - [weakSelf.backupStorage runAsyncRegistrationsWithCompletion:^{ + [storageReference.storage runSyncRegistrations]; + [storageReference.storage runAsyncRegistrationsWithCompletion:^{ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - completion([weakSelf exportDatabaseContents]); + [weakSelf exportDatabaseContentsAndCleanup:storageReference completion:completion]; }); }]; }); } -- (BOOL)exportDatabaseContents +- (void)exportDatabaseContentsAndCleanup:(OWSStorageReference *)storageReference + completion:(OWSBackupJobBoolCompletion)completion { + OWSAssert(storageReference); + OWSAssert(completion); + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - YapDatabaseConnection *_Nullable tempDBConnection = self.backupStorage.newDatabaseConnection; + __weak YapDatabase *_Nullable weakDatabase = nil; + dispatch_queue_t snapshotQueue; + dispatch_queue_t writeQueue; + NSArray *_Nullable allDatabaseFilePaths = nil; + + @autoreleasepool { + allDatabaseFilePaths = [self exportDatabaseContents:storageReference]; + if (!allDatabaseFilePaths) { + completion(NO); + } + + // After the data has been written to the database snapshot, + // we need to synchronously block until the database has been + // completely closed. This is non-trivial because the database + // does a bunch of async work as its closing. + YapDatabase *database = storageReference.storage.database; + + weakDatabase = database; + snapshotQueue = database->snapshotQueue; + writeQueue = database->writeQueue; + + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_DATABASE_FINALIZED", + @"Indicates that the backup export data is being finalized.") + progress:nil]; + + // Flush these two queues immediately. + [database flushInternalQueue]; + [database flushCheckpointQueue]; + + // Close the database. + storageReference.storage = nil; + } + + // Flush these queues, which may contain lingering + // references to the database. + dispatch_sync(snapshotQueue, + ^{ + }); + dispatch_sync(writeQueue, + ^{ + }); + + // YapDatabase retains the registration connection for N seconds. + // The conneciton retains a strong reference to the database. + // We therefore need to wait a bit longer to ensure that this + // doesn't block deallocation. + NSTimeInterval kRegistrationConnectionDelaySeconds = 5.0 * 1.2; + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(kRegistrationConnectionDelaySeconds * NSEC_PER_SEC)), + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), + ^{ + // Dispatch to main thread to wait for any lingering notifications fired by + // database (e.g. cross process notifier). + dispatch_async(dispatch_get_main_queue(), ^{ + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + // Verify that the database is indeed closed. + YapDatabase *_Nullable strongDatabase = weakDatabase; + OWSAssert(!strongDatabase); + + // Capture the list of database files to save. + NSMutableArray *databaseFilePaths = [NSMutableArray new]; + for (NSString *filePath in allDatabaseFilePaths) { + if ([[NSFileManager defaultManager] fileExistsAtPath:filePath]) { + [databaseFilePaths addObject:filePath]; + } + } + if (databaseFilePaths.count < 1) { + OWSProdLogAndFail(@"%@ Can't find database file.", self.logTag); + return completion(NO); + } + self.databaseFilePaths = [databaseFilePaths mutableCopy]; + + completion(YES); + }); + }); + }); +} + +- (nullable NSArray *)exportDatabaseContents:(OWSStorageReference *)storageReference +{ + OWSAssert(storageReference); + + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_DATABASE_EXPORT", + @"Indicates that the database data is being exported.") + progress:nil]; + + YapDatabaseConnection *_Nullable tempDBConnection = storageReference.storage.newDatabaseConnection; if (!tempDBConnection) { OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag); - return NO; + return nil; } YapDatabaseConnection *_Nullable primaryDBConnection = self.primaryStorage.newDatabaseConnection; if (!primaryDBConnection) { OWSProdLogAndFail(@"%@ Could not create primaryDBConnection.", self.logTag); - return NO; + return nil; } + NSString *const kDatabaseSnapshotFilename_Threads = @"threads"; + NSString *const kDatabaseSnapshotFilename_Interactions = @"interactions"; + NSString *const kDatabaseSnapshotFilename_Attachments = @"attachments"; + NSString *const kDatabaseSnapshotFilename_Migrations = @"migrations"; + OWSExportStream *_Nullable exportStream_Threads = + [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Threads jobTempDirPath:self.jobTempDirPath]; + OWSExportStream *_Nullable exportStream_Interactions = + [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Interactions + jobTempDirPath:self.jobTempDirPath]; + OWSExportStream *_Nullable exportStream_Attachments = + [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Attachments jobTempDirPath:self.jobTempDirPath]; + OWSExportStream *_Nullable exportStream_Migrations = + [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Migrations jobTempDirPath:self.jobTempDirPath]; + if (!(exportStream_Threads && exportStream_Interactions && exportStream_Attachments && exportStream_Migrations)) { + return nil; + } + NSArray *exportStreams = @[ + exportStream_Threads, + exportStream_Interactions, + exportStream_Attachments, + exportStream_Migrations, + ]; + __block unsigned long long copiedThreads = 0; __block unsigned long long copiedInteractions = 0; __block unsigned long long copiedEntities = 0; @@ -280,6 +551,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe self.attachmentFilePathMap = [NSMutableDictionary new]; + __block BOOL aborted = NO; [primaryDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) { [tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { [dstTransaction setObject:@(YES) @@ -303,6 +575,12 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [thread saveWithTransaction:dstTransaction]; copiedThreads++; copiedEntities++; + + if (![exportStream_Threads writeObject:thread]) { + *stop = YES; + aborted = YES; + return; + } }]; // Copy attachments. @@ -330,6 +608,12 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [attachment saveWithTransaction:dstTransaction]; copiedAttachments++; copiedEntities++; + + if (![exportStream_Attachments writeObject:attachment]) { + *stop = YES; + aborted = YES; + return; + } }]; // Copy interactions. @@ -362,6 +646,12 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [interaction saveWithTransaction:dstTransaction]; copiedInteractions++; copiedEntities++; + + if (![exportStream_Interactions writeObject:interaction]) { + *stop = YES; + aborted = YES; + return; + } }]; // Copy migrations. @@ -381,12 +671,36 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [migration saveWithTransaction:dstTransaction]; copiedMigrations++; copiedEntities++; + + if (![exportStream_Migrations writeObject:migration]) { + *stop = YES; + aborted = YES; + return; + } }]; }]; }]; + unsigned long long totalZipFileSize = 0; + for (OWSExportStream *exportStream in exportStreams) { + if (![exportStream closeAndZipData]) { + DDLogError(@"%@ couldn't close database snapshot zip.", self.logTag); + return nil; + } + NSNumber *_Nullable fileSize = [OWSFileSystem fileSizeOfPath:exportStream.zipFilePath]; + if (!fileSize) { + DDLogError(@"%@ couldn't get file size of database snapshot zip.", self.logTag); + return nil; + } + totalZipFileSize += fileSize.unsignedLongLongValue; + } + + if (aborted) { + return nil; + } + if (self.isComplete) { - return NO; + return nil; } // TODO: Should we do a database checkpoint? @@ -396,19 +710,44 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe DDLogInfo(@"%@ copiedAttachments: %llu", self.logTag, copiedAttachments); DDLogInfo(@"%@ copiedMigrations: %llu", self.logTag, copiedMigrations); - [self.backupStorage logFileSizes]; + [storageReference.storage logFileSizes]; + + unsigned long long totalDbFileSize + = ([OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath].unsignedLongLongValue + + [OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath_WAL].unsignedLongLongValue + + [OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath_SHM].unsignedLongLongValue); + if (totalZipFileSize > 0 && totalDbFileSize > 0) { + DDLogInfo(@"%@ file size savings: %llu / %llu = %0.2f", + self.logTag, + totalZipFileSize, + totalDbFileSize, + totalZipFileSize / (CGFloat)totalDbFileSize); + } // Capture the list of files to save. - self.databaseFilePaths = [@[ - self.backupStorage.databaseFilePath, - self.backupStorage.databaseFilePath_WAL, - self.backupStorage.databaseFilePath_SHM, - ] mutableCopy]; + return @[ + storageReference.storage.databaseFilePath, + storageReference.storage.databaseFilePath_WAL, + storageReference.storage.databaseFilePath_SHM, + ]; +} - // Close the database. - tempDBConnection = nil; - self.backupStorage = nil; +- (BOOL)writeObject:(TSYapDatabaseObject *)object fileHandle:(NSFileHandle *)fileHandle +{ + OWSAssert(object); + OWSAssert(fileHandle); + NSData *_Nullable data = [NSKeyedArchiver archivedDataWithRootObject:object]; + if (!data) { + OWSProdLogAndFail(@"%@ couldn't serialize database object: %@", self.logTag, [object class]); + return NO; + } + + // We use a fixed width data type. + unsigned int dataLength = (unsigned int)data.length; + NSData *dataLengthData = [NSData dataWithBytes:&dataLength length:sizeof(dataLength)]; + [fileHandle writeData:dataLengthData]; + [fileHandle writeData:data]; return YES; } @@ -477,6 +816,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe } failure:^(NSError *error) { // Database files are critical so any error uploading them is unrecoverable. + DDLogVerbose(@"%@ error while saving file: %@", self.logTag, filePath); completion(error); }]; return YES; diff --git a/Signal/translations/en.lproj/Localizable.strings b/Signal/translations/en.lproj/Localizable.strings index e2e2de31c0..a20dbba54c 100644 --- a/Signal/translations/en.lproj/Localizable.strings +++ b/Signal/translations/en.lproj/Localizable.strings @@ -161,7 +161,7 @@ "BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT" = "Backup data could be exported."; /* Error indicating that the app received an invalid response from CloudKit. */ -"BACKUP_EXPORT_ERROR_INVALID_CLOUDKIT_RESPONSE" = "Invalid server response."; +"BACKUP_EXPORT_ERROR_INVALID_CLOUDKIT_RESPONSE" = "Invalid Service Response"; /* Error indicating the a backup export failed to save a file to the cloud. */ "BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED" = "Backup could not upload data."; @@ -172,6 +172,12 @@ /* Indicates that the backup export is being configured. */ "BACKUP_EXPORT_PHASE_CONFIGURATION" = "Initializing Backup"; +/* Indicates that the database data is being exported. */ +"BACKUP_EXPORT_PHASE_DATABASE_EXPORT" = "Exporting Data"; + +/* Indicates that the backup export data is being finalized. */ +"BACKUP_EXPORT_PHASE_DATABASE_FINALIZED" = "Finalizing Data"; + /* Indicates that the backup export data is being exported. */ "BACKUP_EXPORT_PHASE_EXPORT" = "Exporting Backup"; diff --git a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m index 3d575e4f7d..e059475679 100644 --- a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m +++ b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m @@ -326,6 +326,16 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage) return OWSPrimaryStorage.databaseFilePath; } +- (NSString *)databaseFilePath_SHM +{ + return OWSPrimaryStorage.databaseFilePath_SHM; +} + +- (NSString *)databaseFilePath_WAL +{ + return OWSPrimaryStorage.databaseFilePath_WAL; +} + - (NSString *)databaseFilename_SHM { return OWSPrimaryStorage.databaseFilename_SHM; diff --git a/SignalServiceKit/src/Storage/OWSStorage+Subclass.h b/SignalServiceKit/src/Storage/OWSStorage+Subclass.h index 1417b4fb20..4a4232c290 100644 --- a/SignalServiceKit/src/Storage/OWSStorage+Subclass.h +++ b/SignalServiceKit/src/Storage/OWSStorage+Subclass.h @@ -6,8 +6,12 @@ NS_ASSUME_NONNULL_BEGIN +@class YapDatabase; + @interface OWSStorage (Subclass) +@property (atomic, nullable, readonly) YapDatabase *database; + - (void)loadDatabase; - (void)runSyncRegistrations; diff --git a/SignalServiceKit/src/Storage/OWSStorage.h b/SignalServiceKit/src/Storage/OWSStorage.h index 4e0857cb51..75f710db16 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.h +++ b/SignalServiceKit/src/Storage/OWSStorage.h @@ -61,6 +61,8 @@ extern NSString *const StorageIsReadyNotification; - (nullable id)registeredExtension:(NSString *)extensionName; - (unsigned long long)databaseFileSize; +- (unsigned long long)databaseWALFileSize; +- (unsigned long long)databaseSHMFileSize; - (YapDatabaseConnection *)registrationConnection; diff --git a/SignalServiceKit/src/Storage/OWSStorage.m b/SignalServiceKit/src/Storage/OWSStorage.m index 0b84a0e780..641749cfbc 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.m +++ b/SignalServiceKit/src/Storage/OWSStorage.m @@ -46,6 +46,14 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); #pragma mark - +//@interface OWSDatabaseConnection : YapDatabaseConnection +// +//@property (atomic, weak) id delegate; +// +//@end + +#pragma mark - + @implementation OWSDatabaseConnection - (id)initWithDatabase:(YapDatabase *)database delegate:(id)delegate @@ -58,7 +66,7 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); OWSAssert(delegate); - _delegate = delegate; + self.delegate = delegate; return self; } @@ -134,9 +142,8 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); @property (atomic, weak) id delegate; -@property (atomic, nullable) YapDatabaseConnection *registrationConnectionCached; - - (instancetype)init NS_UNAVAILABLE; + - (id)initWithPath:(NSString *)inPath serializer:(nullable YapDatabaseSerializer)inSerializer deserializer:(YapDatabaseDeserializer)inDeserializer @@ -163,7 +170,7 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); OWSAssert(delegate); - _delegate = delegate; + self.delegate = delegate; return self; } @@ -184,21 +191,15 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); - (YapDatabaseConnection *)registrationConnection { - @synchronized(self) - { - if (!self.registrationConnectionCached) { - YapDatabaseConnection *connection = [super registrationConnection]; + YapDatabaseConnection *connection = [super registrationConnection]; #ifdef DEBUG - // Flag the registration connection as such. - OWSAssert([connection isKindOfClass:[OWSDatabaseConnection class]]); - ((OWSDatabaseConnection *)connection).canWriteBeforeStorageReady = YES; + // Flag the registration connection as such. + OWSAssert([connection isKindOfClass:[OWSDatabaseConnection class]]); + ((OWSDatabaseConnection *)connection).canWriteBeforeStorageReady = YES; #endif - self.registrationConnectionCached = connection; - } - return self.registrationConnectionCached; - } + return connection; } @end @@ -278,6 +279,9 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); - (void)dealloc { + // Surface memory leaks by logging the deallocation of this class. + DDLogVerbose(@"Dealloc: %@", self.class); + [[NSNotificationCenter defaultCenter] removeObserver:self]; } @@ -363,11 +367,6 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); for (OWSStorage *storage in self.allStorages) { [storage runAsyncRegistrationsWithCompletion:^{ if ([self postRegistrationCompleteNotificationIfPossible]) { - // If all registrations are complete, clean up the - // registration process. - - ((OWSDatabase *)storage.database).registrationConnectionCached = nil; - backgroundTask = nil; } }]; @@ -408,15 +407,23 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); - (BOOL)tryToLoadDatabase { + __weak OWSStorage *weakSelf = self; + YapDatabaseOptions *options = [[YapDatabaseOptions alloc] init]; options.corruptAction = YapDatabaseCorruptAction_Fail; options.enableMultiProcessSupport = YES; options.cipherKeySpecBlock = ^{ + // NOTE: It's critical that we don't capture a reference to self + // (e.g. by using OWSAssert()) or this database will contain a + // circular reference and will leak. + OWSStorage *strongSelf = weakSelf; + OWSCAssert(strongSelf); + // Rather than compute this once and capture the value of the key // in the closure, we prefer to fetch the key from the keychain multiple times // in order to keep the key out of application memory. - NSData *databaseKeySpec = [self databaseKeySpec]; - OWSAssert(databaseKeySpec.length == kSQLCipherKeySpecLength); + NSData *databaseKeySpec = [strongSelf databaseKeySpec]; + OWSCAssert(databaseKeySpec.length == kSQLCipherKeySpecLength); return databaseKeySpec; }; @@ -709,6 +716,16 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); return [OWSFileSystem fileSizeOfPath:self.databaseFilePath].unsignedLongLongValue; } +- (unsigned long long)databaseWALFileSize +{ + return [OWSFileSystem fileSizeOfPath:self.databaseFilePath_WAL].unsignedLongLongValue; +} + +- (unsigned long long)databaseSHMFileSize +{ + return [OWSFileSystem fileSizeOfPath:self.databaseFilePath_SHM].unsignedLongLongValue; +} + + (nullable NSData *)tryToLoadKeyChainValue:(NSString *)keychainKey errorHandle:(NSError **)errorHandle { OWSAssert(keychainKey.length > 0); From ca7c75a0817a68855354db147a5203de0a2b6650 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Fri, 16 Mar 2018 11:55:16 -0300 Subject: [PATCH 02/13] Rework database snapshot representation, encryption, etc. --- Signal.xcodeproj/project.pbxproj | 6 + Signal/src/util/OWSBackup.m | 31 +- Signal/src/util/OWSBackupEncryption.h | 42 + Signal/src/util/OWSBackupEncryption.m | 139 +++ Signal/src/util/OWSBackupExportJob.m | 1022 +++++++---------- Signal/src/util/OWSBackupImportJob.m | 1 + Signal/src/util/OWSBackupJob.h | 21 +- Signal/src/util/OWSBackupJob.m | 57 +- .../protobuf/OWSSignalServiceProtos.proto | 18 + .../src/Messages/OWSSignalServiceProtos.pb.h | 125 ++ .../src/Messages/OWSSignalServiceProtos.pb.m | 510 ++++++++ 11 files changed, 1283 insertions(+), 689 deletions(-) create mode 100644 Signal/src/util/OWSBackupEncryption.h create mode 100644 Signal/src/util/OWSBackupEncryption.m diff --git a/Signal.xcodeproj/project.pbxproj b/Signal.xcodeproj/project.pbxproj index 4b116e5de0..2ebc0623e9 100644 --- a/Signal.xcodeproj/project.pbxproj +++ b/Signal.xcodeproj/project.pbxproj @@ -40,6 +40,7 @@ 340FC8C7204DE64D007AEB0F /* OWSBackupAPI.swift in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8C6204DE64D007AEB0F /* OWSBackupAPI.swift */; }; 340FC8CA20517B84007AEB0F /* OWSBackupImportJob.m in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8C820517B84007AEB0F /* OWSBackupImportJob.m */; }; 340FC8CD20518C77007AEB0F /* OWSBackupJob.m in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8CC20518C76007AEB0F /* OWSBackupJob.m */; }; + 340FC8D0205BF2FA007AEB0F /* OWSBackupEncryption.m in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8CE205BF2FA007AEB0F /* OWSBackupEncryption.m */; }; 341F2C0F1F2B8AE700D07D6B /* DebugUIMisc.m in Sources */ = {isa = PBXBuildFile; fileRef = 341F2C0E1F2B8AE700D07D6B /* DebugUIMisc.m */; }; 3430FE181F7751D4000EC51B /* GiphyAPI.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3430FE171F7751D4000EC51B /* GiphyAPI.swift */; }; 34330A5A1E7875FB00DF2FB9 /* fontawesome-webfont.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 34330A591E7875FB00DF2FB9 /* fontawesome-webfont.ttf */; }; @@ -588,6 +589,8 @@ 340FC8C920517B84007AEB0F /* OWSBackupImportJob.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSBackupImportJob.h; sourceTree = ""; }; 340FC8CB20518C76007AEB0F /* OWSBackupJob.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSBackupJob.h; sourceTree = ""; }; 340FC8CC20518C76007AEB0F /* OWSBackupJob.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSBackupJob.m; sourceTree = ""; }; + 340FC8CE205BF2FA007AEB0F /* OWSBackupEncryption.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSBackupEncryption.m; sourceTree = ""; }; + 340FC8CF205BF2FA007AEB0F /* OWSBackupEncryption.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSBackupEncryption.h; sourceTree = ""; }; 341458471FBE11C4005ABCF9 /* fa */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = fa; path = translations/fa.lproj/Localizable.strings; sourceTree = ""; }; 341F2C0D1F2B8AE700D07D6B /* DebugUIMisc.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DebugUIMisc.h; sourceTree = ""; }; 341F2C0E1F2B8AE700D07D6B /* DebugUIMisc.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = DebugUIMisc.m; sourceTree = ""; }; @@ -1955,6 +1958,8 @@ 34A9105E1FFEB113000C4745 /* OWSBackup.h */, 34A9105F1FFEB114000C4745 /* OWSBackup.m */, 340FC8C6204DE64D007AEB0F /* OWSBackupAPI.swift */, + 340FC8CF205BF2FA007AEB0F /* OWSBackupEncryption.h */, + 340FC8CE205BF2FA007AEB0F /* OWSBackupEncryption.m */, 340FC8BE204DB7D1007AEB0F /* OWSBackupExportJob.h */, 340FC8BF204DB7D2007AEB0F /* OWSBackupExportJob.m */, 340FC8C920517B84007AEB0F /* OWSBackupImportJob.h */, @@ -3172,6 +3177,7 @@ 34D1F0B71F87F8850066283D /* OWSGenericAttachmentView.m in Sources */, 34B3F8801E8DF1700035BE1A /* InviteFlow.swift in Sources */, 457C87B82032645C008D52D6 /* DebugUINotifications.swift in Sources */, + 340FC8D0205BF2FA007AEB0F /* OWSBackupEncryption.m in Sources */, 458E38371D668EBF0094BD24 /* OWSDeviceProvisioningURLParser.m in Sources */, 4517642B1DE939FD00EDB8B9 /* ContactCell.swift in Sources */, 340FC8AB204DAC8D007AEB0F /* DomainFrontingCountryViewController.m in Sources */, diff --git a/Signal/src/util/OWSBackup.m b/Signal/src/util/OWSBackup.m index d05368211b..504e1abd37 100644 --- a/Signal/src/util/OWSBackup.m +++ b/Signal/src/util/OWSBackup.m @@ -13,7 +13,6 @@ NSString *const NSNotificationNameBackupStateDidChange = @"NSNotificationNameBac NSString *const OWSPrimaryStorage_OWSBackupCollection = @"OWSPrimaryStorage_OWSBackupCollection"; NSString *const OWSBackup_IsBackupEnabledKey = @"OWSBackup_IsBackupEnabledKey"; -NSString *const OWSBackup_BackupKeyKey = @"OWSBackup_BackupKeyKey"; NSString *const OWSBackup_LastExportSuccessDateKey = @"OWSBackup_LastExportSuccessDateKey"; NSString *const OWSBackup_LastExportFailureDateKey = @"OWSBackup_LastExportFailureDateKey"; @@ -107,31 +106,6 @@ NS_ASSUME_NONNULL_BEGIN }); } -- (void)setBackupPrivateKey:(NSData *)value -{ - OWSAssert(value); - - // TODO: Use actual key. - [self.dbConnection setObject:value - forKey:OWSBackup_BackupKeyKey - inCollection:OWSPrimaryStorage_OWSBackupCollection]; -} - -- (nullable NSData *)backupPrivateKey -{ - NSData *_Nullable result = - [self.dbConnection objectForKey:OWSBackup_BackupKeyKey inCollection:OWSPrimaryStorage_OWSBackupCollection]; - if (!result) { - // TODO: Use actual key. - const NSUInteger kBackupPrivateKeyLength = 32; - result = [Randomness generateRandomBytes:kBackupPrivateKeyLength]; - [self setBackupPrivateKey:result]; - } - OWSAssert(result); - OWSAssert([result isKindOfClass:[NSData class]]); - return result; -} - #pragma mark - Backup Export - (void)tryToExportBackup @@ -383,9 +357,10 @@ NS_ASSUME_NONNULL_BEGIN #pragma mark - OWSBackupJobDelegate // We use a delegate method to avoid storing this key in memory. -- (nullable NSData *)backupKey +- (nullable NSData *)backupEncryptionKey { - return self.backupPrivateKey; + // TODO: Use actual encryption key. + return [@"temp" dataUsingEncoding:NSUTF8StringEncoding]; } - (void)backupJobDidSucceed:(OWSBackupJob *)backupJob diff --git a/Signal/src/util/OWSBackupEncryption.h b/Signal/src/util/OWSBackupEncryption.h new file mode 100644 index 0000000000..5729f59979 --- /dev/null +++ b/Signal/src/util/OWSBackupEncryption.h @@ -0,0 +1,42 @@ +// +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. +// + +NS_ASSUME_NONNULL_BEGIN + +@interface OWSBackupEncryptedItem : NSObject + +@property (nonatomic) NSString *filePath; + +@property (nonatomic) NSData *encryptionKey; + +@end + +#pragma mark - + +@interface OWSBackupEncryption : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +- (instancetype)initWithJobTempDirPath:(NSString *)jobTempDirPath; + +#pragma mark - Encrypt + +- (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath; + +- (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath + encryptionKey:(NSData *)encryptionKey; + +- (nullable OWSBackupEncryptedItem *)encryptDataAsTempFile:(NSData *)srcData; + +- (nullable OWSBackupEncryptedItem *)encryptDataAsTempFile:(NSData *)srcData encryptionKey:(NSData *)encryptionKey; + +#pragma mark - Decrypt + +- (nullable NSString *)decryptFileAsTempFile:(NSString *)srcFilePath encryptionKey:(NSData *)encryptionKey; + +- (nullable NSData *)decryptDataAsData:(NSData *)srcData encryptionKey:(NSData *)encryptionKey; + +@end + +NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackupEncryption.m b/Signal/src/util/OWSBackupEncryption.m new file mode 100644 index 0000000000..8b9ecd051f --- /dev/null +++ b/Signal/src/util/OWSBackupEncryption.m @@ -0,0 +1,139 @@ +// +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. +// + +#import "OWSBackupEncryption.h" +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +// TODO: +static const NSUInteger kOWSBackupKeyLength = 32; + +@implementation OWSBackupEncryptedItem + +@end + +#pragma mark - + +@interface OWSBackupEncryption () + +@property (nonatomic) NSString *jobTempDirPath; + +@end + +#pragma mark - + +@implementation OWSBackupEncryption + +- (instancetype)initWithJobTempDirPath:(NSString *)jobTempDirPath +{ + if (!(self = [super init])) { + return self; + } + + self.jobTempDirPath = jobTempDirPath; + + return self; +} + +#pragma mark - Encrypt + +- (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath +{ + OWSAssert(srcFilePath.length > 0); + + NSData *encryptionKey = [Randomness generateRandomBytes:(int)kOWSBackupKeyLength]; + + return [self encryptFileAsTempFile:srcFilePath encryptionKey:encryptionKey]; +} + +- (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath encryptionKey:(NSData *)encryptionKey +{ + OWSAssert(srcFilePath.length > 0); + OWSAssert(encryptionKey.length > 0); + + // TODO: Encrypt the file without loading it into memory. + NSData *_Nullable srcData = [NSData dataWithContentsOfFile:srcFilePath]; + if (!srcData) { + OWSProdLogAndFail(@"%@ could not load file into memory", self.logTag); + return nil; + } + return [self encryptDataAsTempFile:srcData encryptionKey:encryptionKey]; +} + +- (nullable OWSBackupEncryptedItem *)encryptDataAsTempFile:(NSData *)srcData +{ + OWSAssert(srcData); + + NSData *encryptionKey = [Randomness generateRandomBytes:(int)kOWSBackupKeyLength]; + + return [self encryptDataAsTempFile:srcData encryptionKey:encryptionKey]; +} + +- (nullable OWSBackupEncryptedItem *)encryptDataAsTempFile:(NSData *)srcData encryptionKey:(NSData *)encryptionKey +{ + OWSAssert(srcData); + OWSAssert(encryptionKey.length > 0); + + // TODO: Encrypt the data using key; + + NSString *dstFilePath = [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; + NSError *error; + BOOL success = [srcData writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; + if (!success || error) { + OWSProdLogAndFail(@"%@ error writing encrypted data: %@", self.logTag, error); + return nil; + } + [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; + OWSBackupEncryptedItem *item = [OWSBackupEncryptedItem new]; + item.filePath = dstFilePath; + item.encryptionKey = encryptionKey; + return item; +} + +#pragma mark - Decrypt + +- (nullable NSString *)decryptFileAsTempFile:(NSString *)srcFilePath encryptionKey:(NSData *)encryptionKey +{ + OWSAssert(srcFilePath.length > 0); + OWSAssert(encryptionKey.length > 0); + + // TODO: Decrypt the file without loading it into memory. + NSData *_Nullable srcData = [NSData dataWithContentsOfFile:srcFilePath]; + if (!srcData) { + OWSProdLogAndFail(@"%@ could not load file into memory", self.logTag); + return nil; + } + + NSData *_Nullable dstData = [self decryptDataAsData:srcData encryptionKey:encryptionKey]; + if (!dstData) { + return nil; + } + + NSString *dstFilePath = [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; + NSError *error; + BOOL success = [dstData writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; + if (!success || error) { + OWSProdLogAndFail(@"%@ error writing decrypted data: %@", self.logTag, error); + return nil; + } + [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; + + return dstFilePath; +} + +- (nullable NSData *)decryptDataAsData:(NSData *)srcData encryptionKey:(NSData *)encryptionKey +{ + OWSAssert(srcData); + OWSAssert(encryptionKey.length > 0); + + // TODO: Decrypt the data using key; + + return srcData; +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index 8570da789a..b832bae455 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -4,9 +4,8 @@ #import "OWSBackupExportJob.h" #import "OWSDatabaseMigration.h" +#import "OWSSignalServiceProtos.pb.h" #import "Signal-Swift.h" -#import "zlib.h" -#import #import #import #import @@ -18,106 +17,92 @@ #import #import #import -#import -#import -#import + +@import Compression; +#import "OWSBackupEncryption.h" + +// TODO: NS_ASSUME_NONNULL_BEGIN -NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKeySpec"; +@class OWSAttachmentExport; -@interface YapDatabase (OWSBackupExportJob) +@interface OWSBackupExportItem : NSObject -- (void)flushInternalQueue; -- (void)flushCheckpointQueue; +@property (nonatomic) OWSBackupEncryptedItem *encryptedItem; + +@property (nonatomic) NSString *recordName; + +// This property is optional and represents the location of this +// item relative to the root directory for items of this type. +//@property (nonatomic, nullable) NSString *fileRelativePath; + +// This property is optional and is only used for attachments. +@property (nonatomic, nullable) OWSAttachmentExport *attachmentExport; + +- (instancetype)init NS_UNAVAILABLE; @end #pragma mark - -@interface OWSStorageReference : NSObject +@implementation OWSBackupExportItem -@property (nonatomic, nullable) OWSStorage *storage; - -@end - -#pragma mark - - -@implementation OWSStorageReference - -@end - -#pragma mark - - -// TODO: This implementation is a proof-of-concept and -// isn't production ready. -@interface OWSExportStream : NSObject - -@property (nonatomic) NSString *dataFilePath; - -@property (nonatomic) NSString *zipFilePath; - -@property (nonatomic, nullable) NSFileHandle *fileHandle; - -@property (nonatomic, nullable) SSZipArchive *zipFile; - -@end - -#pragma mark - - -@implementation OWSExportStream - -- (void)dealloc +- (instancetype)initWithEncryptedItem:(OWSBackupEncryptedItem *)encryptedItem { - // Surface memory leaks by logging the deallocation of view controllers. - DDLogVerbose(@"Dealloc: %@", self.class); - - [self.fileHandle closeFile]; - - if (self.zipFile) { - if (![self.zipFile close]) { - DDLogError(@"%@ couldn't close to database snapshot zip.", self.logTag); - } + if (!(self = [super init])) { + return self; } + + OWSAssert(encryptedItem); + + self.encryptedItem = encryptedItem; + + return self; } -+ (OWSExportStream *)exportStreamWithName:(NSString *)filename jobTempDirPath:(NSString *)jobTempDirPath -{ - OWSAssert(filename.length > 0); - OWSAssert(jobTempDirPath.length > 0); +@end - OWSExportStream *exportStream = [OWSExportStream new]; - exportStream.dataFilePath = [jobTempDirPath stringByAppendingPathComponent:filename]; - exportStream.zipFilePath = [exportStream.dataFilePath stringByAppendingPathExtension:@"zip"]; - if (![exportStream open]) { - return nil; - } - return exportStream; -} +#pragma mark - -- (BOOL)open +@interface OWSDBExportStream : NSObject + +@property (nonatomic) OWSBackupEncryption *encryption; + +@property (nonatomic) NSMutableArray *encryptedItems; + +@property (nonatomic, nullable) OWSSignalServiceProtosBackupSnapshotBuilder *backupSnapshotBuilder; + +@property (nonatomic) NSUInteger cachedItemCount; + +@property (nonatomic) NSUInteger totalItemCount; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +#pragma mark - + +@implementation OWSDBExportStream + +- (instancetype)initWithEncryption:(OWSBackupEncryption *)encryption { - if (![[NSFileManager defaultManager] createFileAtPath:self.dataFilePath contents:nil attributes:nil]) { - OWSProdLogAndFail(@"%@ Could not create database snapshot stream.", self.logTag); - return NO; + if (!(self = [super init])) { + return self; } - if (![OWSFileSystem protectFileOrFolderAtPath:self.dataFilePath]) { - OWSProdLogAndFail(@"%@ Could not protect database snapshot stream.", self.logTag); - return NO; - } - NSError *error; - self.fileHandle = [NSFileHandle fileHandleForWritingToURL:[NSURL fileURLWithPath:self.dataFilePath] error:&error]; - if (!self.fileHandle || error) { - OWSProdLogAndFail(@"%@ Could not open database snapshot stream: %@.", self.logTag, error); - return NO; - } - return YES; + + OWSAssert(encryption); + + self.encryptedItems = [NSMutableArray new]; + self.encryption = encryption; + + return self; } - (BOOL)writeObject:(TSYapDatabaseObject *)object + entityType:(OWSSignalServiceProtosBackupSnapshotBackupEntityType)entityType { OWSAssert(object); - OWSAssert(self.fileHandle); NSData *_Nullable data = [NSKeyedArchiver archivedDataWithRootObject:object]; if (!data) { @@ -125,55 +110,68 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe return NO; } - // We use a fixed width data type. - unsigned int dataLength = (unsigned int)data.length; - NSData *dataLengthData = [NSData dataWithBytes:&dataLength length:sizeof(dataLength)]; - [self.fileHandle writeData:dataLengthData]; - [self.fileHandle writeData:data]; + if (!self.backupSnapshotBuilder) { + self.backupSnapshotBuilder = [OWSSignalServiceProtosBackupSnapshotBuilder new]; + } + + OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder *entityBuilder = + [OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder new]; + [entityBuilder setType:entityType]; + [entityBuilder setData:data]; + + [self.backupSnapshotBuilder addEntity:[entityBuilder build]]; + + self.cachedItemCount = self.cachedItemCount + 1; + self.totalItemCount = self.totalItemCount + 1; + + static const int kMaxDBSnapshotSize = 1000; + if (self.cachedItemCount > kMaxDBSnapshotSize) { + return [self flush]; + } + return YES; } -- (BOOL)closeAndZipData +// Write cached data to disk, if necessary. +- (BOOL)flush { - [self.fileHandle closeFile]; - self.fileHandle = nil; - - self.zipFile = [[SSZipArchive alloc] initWithPath:self.zipFilePath]; - if (!self.zipFile) { - OWSProdLogAndFail(@"%@ Could not create database snapshot zip.", self.logTag); - return NO; + if (!self.backupSnapshotBuilder) { + return YES; } - if (![self.zipFile open]) { - OWSProdLogAndFail(@"%@ Could not open database snapshot zip.", self.logTag); + + NSData *_Nullable uncompressedData = [self.backupSnapshotBuilder build].data; + self.backupSnapshotBuilder = nil; + self.cachedItemCount = 0; + if (!uncompressedData) { + OWSProdLogAndFail(@"%@ couldn't convert database snapshot to data.", self.logTag); return NO; } - BOOL success = [self.zipFile writeFileAtPath:self.dataFilePath - withFileName:@"payload" - compressionLevel:Z_BEST_COMPRESSION - password:nil - AES:NO]; - if (!success) { - OWSProdLogAndFail(@"%@ Could not write to database snapshot zip.", self.logTag); + size_t srcLength = [uncompressedData length]; + const uint8_t *srcBuffer = (const uint8_t *)[uncompressedData bytes]; + if (!srcBuffer) { return NO; } - - if (![self.zipFile close]) { - DDLogError(@"%@ couldn't close database snapshot zip.", self.logTag); + // This assumes that dst will always be smaller than src. + uint8_t *dstBuffer = malloc(sizeof(uint8_t) * srcLength); + if (!dstBuffer) { return NO; } - self.zipFile = nil; - - if (![OWSFileSystem protectFileOrFolderAtPath:self.zipFilePath]) { - DDLogError(@"%@ could not protect database snapshot zip.", self.logTag); - } - - DDLogInfo(@"%@ wrote database snapshot zip: %@ (%@ -> %@)", + // TODO: Should we use COMPRESSION_LZFSE? + size_t dstLength = compression_encode_buffer(dstBuffer, srcLength, srcBuffer, srcLength, NULL, COMPRESSION_LZFSE); + NSData *compressedData = [NSData dataWithBytes:dstBuffer length:dstLength]; + DDLogVerbose(@"%@ compressed %zd -> %zd = %0.2f", self.logTag, - self.zipFilePath.lastPathComponent, - [OWSFileSystem fileSizeOfPath:self.dataFilePath], - [OWSFileSystem fileSizeOfPath:self.zipFilePath]); + srcLength, + dstLength, + (dstLength > srcLength ? (dstLength / (CGFloat)srcLength) : 0)); + free(dstBuffer); + OWSBackupEncryptedItem *_Nullable encryptedItem = [self.encryption encryptDataAsTempFile:compressedData]; + if (!encryptedItem) { + OWSProdLogAndFail(@"%@ couldn't encrypt database snapshot.", self.logTag); + return NO; + } return YES; } @@ -183,12 +181,13 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe @interface OWSAttachmentExport : NSObject -@property (nonatomic, weak) id delegate; -@property (nonatomic) NSString *jobTempDirPath; +@property (nonatomic) OWSBackupEncryption *encryption; @property (nonatomic) NSString *attachmentId; @property (nonatomic) NSString *attachmentFilePath; -@property (nonatomic, nullable) NSString *tempFilePath; @property (nonatomic, nullable) NSString *relativeFilePath; +@property (nonatomic) OWSBackupEncryptedItem *encryptedItem; + +- (instancetype)init NS_UNAVAILABLE; @end @@ -196,21 +195,34 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe @implementation OWSAttachmentExport +- (instancetype)initWithEncryption:(OWSBackupEncryption *)encryption + attachmentId:(NSString *)attachmentId + attachmentFilePath:(NSString *)attachmentFilePath +{ + if (!(self = [super init])) { + return self; + } + + OWSAssert(encryption); + OWSAssert(attachmentId.length > 0); + OWSAssert(attachmentFilePath.length > 0); + + self.encryption = encryption; + self.attachmentId = attachmentId; + self.attachmentFilePath = attachmentFilePath; + + return self; +} + - (void)dealloc { // Surface memory leaks by logging the deallocation. DDLogVerbose(@"Dealloc: %@", self.class); - - // Delete temporary file ASAP. - if (self.tempFilePath) { - [OWSFileSystem deleteFileIfExists:self.tempFilePath]; - } } -// On success, tempFilePath will be non-nil. -- (void)prepareForUpload +// On success, encryptedItem will be non-nil. +- (BOOL)prepareForUpload { - OWSAssert(self.jobTempDirPath.length > 0); OWSAssert(self.attachmentId.length > 0); OWSAssert(self.attachmentFilePath.length > 0); @@ -218,7 +230,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe if (![self.attachmentFilePath hasPrefix:attachmentsDirPath]) { DDLogError(@"%@ attachment has unexpected path.", self.logTag); OWSFail(@"%@ attachment has unexpected path: %@", self.logTag, self.attachmentFilePath); - return; + return NO; } NSString *relativeFilePath = [self.attachmentFilePath substringFromIndex:attachmentsDirPath.length]; NSString *pathSeparator = @"/"; @@ -227,15 +239,14 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe } self.relativeFilePath = relativeFilePath; - NSString *_Nullable tempFilePath = [OWSBackupExportJob encryptFileAsTempFile:self.attachmentFilePath - jobTempDirPath:self.jobTempDirPath - delegate:self.delegate]; - if (!tempFilePath) { + OWSBackupEncryptedItem *_Nullable encryptedItem = [self.encryption encryptFileAsTempFile:self.attachmentFilePath]; + if (!encryptedItem) { DDLogError(@"%@ attachment could not be encrypted.", self.logTag); OWSFail(@"%@ attachment could not be encrypted: %@", self.logTag, self.attachmentFilePath); - return; + return NO; } - self.tempFilePath = tempFilePath; + self.encryptedItem = encryptedItem; + return YES; } @end @@ -246,17 +257,17 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe @property (nonatomic, nullable) OWSBackgroundTask *backgroundTask; -@property (nonatomic) NSMutableArray *databaseFilePaths; -// A map of "record name"-to-"file name". -@property (nonatomic) NSMutableDictionary *databaseRecordMap; +@property (nonatomic) OWSBackupEncryption *encryption; -// A map of "attachment id"-to-"local file path". -@property (nonatomic) NSMutableDictionary *attachmentFilePathMap; -// A map of "record name"-to-"file relative path". -@property (nonatomic) NSMutableDictionary *attachmentRecordMap; +@property (nonatomic) NSMutableArray *unsavedDatabaseItems; -@property (nonatomic, nullable) NSString *manifestFilePath; -@property (nonatomic, nullable) NSString *manifestRecordName; +@property (nonatomic) NSMutableArray *unsavedAttachmentExports; + +@property (nonatomic) NSMutableArray *savedDatabaseItems; + +@property (nonatomic) NSMutableArray *savedAttachmentItems; + +@property (nonatomic, nullable) OWSBackupExportItem *manifestItem; @end @@ -305,29 +316,26 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_EXPORT", @"Indicates that the backup export data is being exported.") progress:nil]; - [self exportDatabase:^(BOOL exportDatabaseSuccess) { - if (!exportDatabaseSuccess) { - [self failWithErrorDescription: - NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", - @"Error indicating the a backup export could not export the user's data.")]; + if (![self exportDatabase]) { + [self failWithErrorDescription: + NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", + @"Error indicating the a backup export could not export the user's data.")]; + return; + } + if (self.isComplete) { + return; + } + [self saveToCloud:^(NSError *_Nullable saveError) { + if (saveError) { + [weakSelf failWithError:saveError]; return; } - - if (self.isComplete) { - return; - } - [self saveToCloud:^(NSError *_Nullable saveError) { - if (saveError) { - [weakSelf failWithError:saveError]; + [self cleanUpCloud:^(NSError *_Nullable cleanUpError) { + if (cleanUpError) { + [weakSelf failWithError:cleanUpError]; return; } - [self cleanUpCloud:^(NSError *_Nullable cleanUpError) { - if (cleanUpError) { - [weakSelf failWithError:cleanUpError]; - return; - } - [weakSelf succeed]; - }]; + [weakSelf succeed]; }]; }]; }]; @@ -344,6 +352,8 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe return completion(NO); } + self.encryption = [[OWSBackupEncryption alloc] initWithJobTempDirPath:self.jobTempDirPath]; + // We need to verify that we have a valid account. // Otherwise, if we re-register on another device, we // continue to backup on our old device, overwriting @@ -367,141 +377,9 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe }]; } -- (void)exportDatabase:(OWSBackupJobBoolCompletion)completion +- (BOOL)exportDatabase { - OWSAssert(completion); - - DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - - if (![OWSBackupJob generateRandomDatabaseKeySpecWithKeychainKey:kOWSBackup_ExportDatabaseKeySpec]) { - OWSProdLogAndFail(@"%@ Could not generate database key spec for export.", self.logTag); - return completion(NO); - } - - // TODO: Move this work into the "export database" method. - NSString *jobDatabaseDirPath = [self.jobTempDirPath stringByAppendingPathComponent:@"database"]; - if (![OWSFileSystem ensureDirectoryExists:jobDatabaseDirPath]) { - OWSProdLogAndFail(@"%@ Could not create jobDatabaseDirPath.", self.logTag); - return completion(NO); - } - - BackupStorageKeySpecBlock keySpecBlock = ^{ - NSData *_Nullable databaseKeySpec = - [OWSBackupJob loadDatabaseKeySpecWithKeychainKey:kOWSBackup_ExportDatabaseKeySpec]; - if (!databaseKeySpec) { - OWSProdLogAndFail(@"%@ Could not load database keyspec for export.", self.logTag); - } - return databaseKeySpec; - }; - - OWSStorageReference *storageReference = [OWSStorageReference new]; - storageReference.storage = - [[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock]; - if (!storageReference.storage) { - OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag); - return completion(NO); - } - - // TODO: Do we really need to run these registrations on the main thread? - __weak OWSBackupExportJob *weakSelf = self; - dispatch_async(dispatch_get_main_queue(), ^{ - [storageReference.storage runSyncRegistrations]; - [storageReference.storage runAsyncRegistrationsWithCompletion:^{ - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - [weakSelf exportDatabaseContentsAndCleanup:storageReference completion:completion]; - }); - }]; - }); -} - -- (void)exportDatabaseContentsAndCleanup:(OWSStorageReference *)storageReference - completion:(OWSBackupJobBoolCompletion)completion -{ - OWSAssert(storageReference); - OWSAssert(completion); - - DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - - __weak YapDatabase *_Nullable weakDatabase = nil; - dispatch_queue_t snapshotQueue; - dispatch_queue_t writeQueue; - NSArray *_Nullable allDatabaseFilePaths = nil; - - @autoreleasepool { - allDatabaseFilePaths = [self exportDatabaseContents:storageReference]; - if (!allDatabaseFilePaths) { - completion(NO); - } - - // After the data has been written to the database snapshot, - // we need to synchronously block until the database has been - // completely closed. This is non-trivial because the database - // does a bunch of async work as its closing. - YapDatabase *database = storageReference.storage.database; - - weakDatabase = database; - snapshotQueue = database->snapshotQueue; - writeQueue = database->writeQueue; - - [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_DATABASE_FINALIZED", - @"Indicates that the backup export data is being finalized.") - progress:nil]; - - // Flush these two queues immediately. - [database flushInternalQueue]; - [database flushCheckpointQueue]; - - // Close the database. - storageReference.storage = nil; - } - - // Flush these queues, which may contain lingering - // references to the database. - dispatch_sync(snapshotQueue, - ^{ - }); - dispatch_sync(writeQueue, - ^{ - }); - - // YapDatabase retains the registration connection for N seconds. - // The conneciton retains a strong reference to the database. - // We therefore need to wait a bit longer to ensure that this - // doesn't block deallocation. - NSTimeInterval kRegistrationConnectionDelaySeconds = 5.0 * 1.2; - dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(kRegistrationConnectionDelaySeconds * NSEC_PER_SEC)), - dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), - ^{ - // Dispatch to main thread to wait for any lingering notifications fired by - // database (e.g. cross process notifier). - dispatch_async(dispatch_get_main_queue(), ^{ - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - // Verify that the database is indeed closed. - YapDatabase *_Nullable strongDatabase = weakDatabase; - OWSAssert(!strongDatabase); - - // Capture the list of database files to save. - NSMutableArray *databaseFilePaths = [NSMutableArray new]; - for (NSString *filePath in allDatabaseFilePaths) { - if ([[NSFileManager defaultManager] fileExistsAtPath:filePath]) { - [databaseFilePaths addObject:filePath]; - } - } - if (databaseFilePaths.count < 1) { - OWSProdLogAndFail(@"%@ Can't find database file.", self.logTag); - return completion(NO); - } - self.databaseFilePaths = [databaseFilePaths mutableCopy]; - - completion(YES); - }); - }); - }); -} - -- (nullable NSArray *)exportDatabaseContents:(OWSStorageReference *)storageReference -{ - OWSAssert(storageReference); + OWSAssert(self.encryption); DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); @@ -509,256 +387,180 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe @"Indicates that the database data is being exported.") progress:nil]; - YapDatabaseConnection *_Nullable tempDBConnection = storageReference.storage.newDatabaseConnection; - if (!tempDBConnection) { - OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag); - return nil; - } - YapDatabaseConnection *_Nullable primaryDBConnection = self.primaryStorage.newDatabaseConnection; - if (!primaryDBConnection) { - OWSProdLogAndFail(@"%@ Could not create primaryDBConnection.", self.logTag); - return nil; - } - - NSString *const kDatabaseSnapshotFilename_Threads = @"threads"; - NSString *const kDatabaseSnapshotFilename_Interactions = @"interactions"; - NSString *const kDatabaseSnapshotFilename_Attachments = @"attachments"; - NSString *const kDatabaseSnapshotFilename_Migrations = @"migrations"; - OWSExportStream *_Nullable exportStream_Threads = - [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Threads jobTempDirPath:self.jobTempDirPath]; - OWSExportStream *_Nullable exportStream_Interactions = - [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Interactions - jobTempDirPath:self.jobTempDirPath]; - OWSExportStream *_Nullable exportStream_Attachments = - [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Attachments jobTempDirPath:self.jobTempDirPath]; - OWSExportStream *_Nullable exportStream_Migrations = - [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Migrations jobTempDirPath:self.jobTempDirPath]; - if (!(exportStream_Threads && exportStream_Interactions && exportStream_Attachments && exportStream_Migrations)) { - return nil; - } - NSArray *exportStreams = @[ - exportStream_Threads, - exportStream_Interactions, - exportStream_Attachments, - exportStream_Migrations, - ]; - - __block unsigned long long copiedThreads = 0; - __block unsigned long long copiedInteractions = 0; - __block unsigned long long copiedEntities = 0; - __block unsigned long long copiedAttachments = 0; - __block unsigned long long copiedMigrations = 0; - - self.attachmentFilePathMap = [NSMutableDictionary new]; - - __block BOOL aborted = NO; - [primaryDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) { - [tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { - [dstTransaction setObject:@(YES) - forKey:kOWSBackup_Snapshot_ValidKey - inCollection:kOWSBackup_Snapshot_Collection]; - - // Copy threads. - [srcTransaction - enumerateKeysAndObjectsInCollection:[TSThread collection] - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - return; - } - if (![object isKindOfClass:[TSThread class]]) { - OWSProdLogAndFail( - @"%@ unexpected class: %@", self.logTag, [object class]); - return; - } - TSThread *thread = object; - [thread saveWithTransaction:dstTransaction]; - copiedThreads++; - copiedEntities++; - - if (![exportStream_Threads writeObject:thread]) { - *stop = YES; - aborted = YES; - return; - } - }]; - - // Copy attachments. - [srcTransaction - enumerateKeysAndObjectsInCollection:[TSAttachment collection] - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - return; - } - if (![object isKindOfClass:[TSAttachment class]]) { - OWSProdLogAndFail( - @"%@ unexpected class: %@", self.logTag, [object class]); - return; - } - if ([object isKindOfClass:[TSAttachmentStream class]]) { - TSAttachmentStream *attachmentStream = object; - NSString *_Nullable filePath = attachmentStream.filePath; - if (filePath) { - OWSAssert(attachmentStream.uniqueId.length > 0); - self.attachmentFilePathMap[attachmentStream.uniqueId] = filePath; - } - } - TSAttachment *attachment = object; - [attachment saveWithTransaction:dstTransaction]; - copiedAttachments++; - copiedEntities++; - - if (![exportStream_Attachments writeObject:attachment]) { - *stop = YES; - aborted = YES; - return; - } - }]; - - // Copy interactions. - // - // Interactions refer to threads and attachments, so copy the last. - [srcTransaction - enumerateKeysAndObjectsInCollection:[TSInteraction collection] - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - return; - } - if (![object isKindOfClass:[TSInteraction class]]) { - OWSProdLogAndFail( - @"%@ unexpected class: %@", self.logTag, [object class]); - return; - } - // Ignore disappearing messages. - if ([object isKindOfClass:[TSMessage class]]) { - TSMessage *message = object; - if (message.isExpiringMessage) { - return; - } - } - TSInteraction *interaction = object; - // Ignore dynamic interactions. - if (interaction.isDynamicInteraction) { - return; - } - [interaction saveWithTransaction:dstTransaction]; - copiedInteractions++; - copiedEntities++; - - if (![exportStream_Interactions writeObject:interaction]) { - *stop = YES; - aborted = YES; - return; - } - }]; - - // Copy migrations. - [srcTransaction - enumerateKeysAndObjectsInCollection:[OWSDatabaseMigration collection] - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - return; - } - if (![object isKindOfClass:[OWSDatabaseMigration class]]) { - OWSProdLogAndFail( - @"%@ unexpected class: %@", self.logTag, [object class]); - return; - } - OWSDatabaseMigration *migration = object; - [migration saveWithTransaction:dstTransaction]; - copiedMigrations++; - copiedEntities++; - - if (![exportStream_Migrations writeObject:migration]) { - *stop = YES; - aborted = YES; - return; - } - }]; - }]; - }]; - - unsigned long long totalZipFileSize = 0; - for (OWSExportStream *exportStream in exportStreams) { - if (![exportStream closeAndZipData]) { - DDLogError(@"%@ couldn't close database snapshot zip.", self.logTag); - return nil; - } - NSNumber *_Nullable fileSize = [OWSFileSystem fileSizeOfPath:exportStream.zipFilePath]; - if (!fileSize) { - DDLogError(@"%@ couldn't get file size of database snapshot zip.", self.logTag); - return nil; - } - totalZipFileSize += fileSize.unsignedLongLongValue; - } - - if (aborted) { - return nil; - } - - if (self.isComplete) { - return nil; - } - // TODO: Should we do a database checkpoint? - - DDLogInfo(@"%@ copiedThreads: %llu", self.logTag, copiedThreads); - DDLogInfo(@"%@ copiedMessages: %llu", self.logTag, copiedInteractions); - DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities); - DDLogInfo(@"%@ copiedAttachments: %llu", self.logTag, copiedAttachments); - DDLogInfo(@"%@ copiedMigrations: %llu", self.logTag, copiedMigrations); - - [storageReference.storage logFileSizes]; - - unsigned long long totalDbFileSize - = ([OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath].unsignedLongLongValue + - [OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath_WAL].unsignedLongLongValue + - [OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath_SHM].unsignedLongLongValue); - if (totalZipFileSize > 0 && totalDbFileSize > 0) { - DDLogInfo(@"%@ file size savings: %llu / %llu = %0.2f", - self.logTag, - totalZipFileSize, - totalDbFileSize, - totalZipFileSize / (CGFloat)totalDbFileSize); - } - - // Capture the list of files to save. - return @[ - storageReference.storage.databaseFilePath, - storageReference.storage.databaseFilePath_WAL, - storageReference.storage.databaseFilePath_SHM, - ]; -} - -- (BOOL)writeObject:(TSYapDatabaseObject *)object fileHandle:(NSFileHandle *)fileHandle -{ - OWSAssert(object); - OWSAssert(fileHandle); - - NSData *_Nullable data = [NSKeyedArchiver archivedDataWithRootObject:object]; - if (!data) { - OWSProdLogAndFail(@"%@ couldn't serialize database object: %@", self.logTag, [object class]); + YapDatabaseConnection *_Nullable dbConnection = self.primaryStorage.newDatabaseConnection; + if (!dbConnection) { + OWSProdLogAndFail(@"%@ Could not create dbConnection.", self.logTag); return NO; } - // We use a fixed width data type. - unsigned int dataLength = (unsigned int)data.length; - NSData *dataLengthData = [NSData dataWithBytes:&dataLength length:sizeof(dataLength)]; - [fileHandle writeData:dataLengthData]; - [fileHandle writeData:data]; + OWSDBExportStream *exportStream = [[OWSDBExportStream alloc] initWithEncryption:self.encryption]; + + __block BOOL aborted = NO; + typedef BOOL (^EntityFilter)(id object); + typedef NSUInteger (^ExportBlock)(YapDatabaseReadTransaction *, + NSString *, + Class, + EntityFilter _Nullable, + OWSSignalServiceProtosBackupSnapshotBackupEntityType); + ExportBlock exportEntities = ^(YapDatabaseReadTransaction *transaction, + NSString *collection, + Class expectedClass, + EntityFilter _Nullable filter, + OWSSignalServiceProtosBackupSnapshotBackupEntityType entityType) { + __block NSUInteger count = 0; + [transaction + enumerateKeysAndObjectsInCollection:collection + usingBlock:^(NSString *key, id object, BOOL *stop) { + if (self.isComplete) { + *stop = YES; + return; + } + if (filter && !filter(object)) { + return; + } + if (![object isKindOfClass:expectedClass]) { + OWSProdLogAndFail(@"%@ unexpected class: %@", self.logTag, [object class]); + return; + } + TSYapDatabaseObject *entity = object; + count++; + + if (![exportStream writeObject:entity entityType:entityType]) { + *stop = YES; + aborted = YES; + return; + } + }]; + return count; + }; + + __block NSUInteger copiedThreads = 0; + __block NSUInteger copiedInteractions = 0; + __block NSUInteger copiedAttachments = 0; + __block NSUInteger copiedMigrations = 0; + self.unsavedAttachmentExports = [NSMutableArray new]; + [dbConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) { + copiedThreads = exportEntities(transaction, + [TSThread collection], + [TSThread class], + nil, + OWSSignalServiceProtosBackupSnapshotBackupEntityTypeThread); + if (aborted) { + return; + } + + copiedAttachments = exportEntities(transaction, + [TSAttachment collection], + [TSAttachment class], + ^(id object) { + if (![object isKindOfClass:[TSAttachmentStream class]]) { + return NO; + } + TSAttachmentStream *attachmentStream = object; + NSString *_Nullable filePath = attachmentStream.filePath; + if (!filePath) { + DDLogError(@"%@ attachment is missing file.", self.logTag); + return NO; + OWSAssert(attachmentStream.uniqueId.length > 0); + } + + // OWSAttachmentExport is used to lazily write an encrypted copy of the + // attachment to disk. + OWSAttachmentExport *attachmentExport = + [[OWSAttachmentExport alloc] initWithEncryption:self.encryption + attachmentId:attachmentStream.uniqueId + attachmentFilePath:filePath]; + [self.unsavedAttachmentExports addObject:attachmentExport]; + + return YES; + }, + OWSSignalServiceProtosBackupSnapshotBackupEntityTypeAttachment); + if (aborted) { + return; + } + + // Interactions refer to threads and attachments, so copy after them. + copiedInteractions = exportEntities(transaction, + [TSInteraction collection], + [TSInteraction class], + ^(id object) { + // Ignore disappearing messages. + if ([object isKindOfClass:[TSMessage class]]) { + TSMessage *message = object; + if (message.isExpiringMessage) { + return NO; + } + } + TSInteraction *interaction = object; + // Ignore dynamic interactions. + if (interaction.isDynamicInteraction) { + return NO; + } + return YES; + }, + OWSSignalServiceProtosBackupSnapshotBackupEntityTypeInteraction); + if (aborted) { + return; + } + + copiedMigrations = exportEntities(transaction, + [OWSDatabaseMigration collection], + [OWSDatabaseMigration class], + nil, + OWSSignalServiceProtosBackupSnapshotBackupEntityTypeMigration); + }]; + + if (aborted || self.isComplete) { + return NO; + } + + if (![exportStream flush]) { + OWSProdLogAndFail(@"%@ Could not flush database snapshots.", self.logTag); + return NO; + } + + self.unsavedDatabaseItems = + [[self exportItemsForEncryptedItems:exportStream.encryptedItems label:@"database snapshots"] mutableCopy]; + + // TODO: Should we do a database checkpoint? + + DDLogInfo(@"%@ copiedThreads: %zd", self.logTag, copiedThreads); + DDLogInfo(@"%@ copiedMessages: %zd", self.logTag, copiedInteractions); + DDLogInfo(@"%@ copiedAttachments: %zd", self.logTag, copiedAttachments); + DDLogInfo(@"%@ copiedMigrations: %zd", self.logTag, copiedMigrations); + DDLogInfo(@"%@ copiedEntities: %zd", self.logTag, exportStream.totalItemCount); + return YES; } +- (NSArray *)exportItemsForEncryptedItems:(NSArray *)encryptedItems + label:(NSString *)label +{ + OWSAssert(encryptedItems); + OWSAssert(label.length); + + NSMutableArray *exportItems = [NSMutableArray new]; + unsigned long long totalFileSize = 0; + for (OWSBackupEncryptedItem *encryptedItem in encryptedItems) { + OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; + exportItem.encryptedItem = encryptedItem; + [exportItems addObject:exportItem]; + + totalFileSize += [OWSFileSystem fileSizeOfPath:encryptedItem.filePath].unsignedLongLongValue; + } + DDLogInfo(@"%@ exported %@: count: %zd, bytes: %llu.", self.logTag, label, exportItems.count, totalFileSize); + + return exportItems; +} + - (void)saveToCloud:(OWSBackupJobCompletion)completion { OWSAssert(completion); DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - self.databaseRecordMap = [NSMutableDictionary new]; - self.attachmentRecordMap = [NSMutableDictionary new]; + self.savedDatabaseItems = [NSMutableArray new]; + self.savedAttachmentItems = [NSMutableArray new]; [self saveNextFileToCloud:completion]; } @@ -774,8 +576,11 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe return; } - CGFloat progress - = (self.databaseRecordMap.count / (CGFloat)(self.databaseRecordMap.count + self.databaseFilePaths.count)); + // Add one for the manifest + NSUInteger unsavedCount = (self.unsavedDatabaseItems.count + self.unsavedAttachmentExports.count + 1); + NSUInteger savedCount = (self.savedDatabaseItems.count + self.savedAttachmentItems.count); + + CGFloat progress = (savedCount / (CGFloat)(unsavedCount + savedCount)); [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_UPLOAD", @"Indicates that the backup export data is being uploaded.") progress:@(progress)]; @@ -789,64 +594,66 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [self saveManifestFileToCloud:completion]; } +// This method returns YES IFF "work was done and there might be more work to do". - (BOOL)saveNextDatabaseFileToCloud:(OWSBackupJobCompletion)completion { OWSAssert(completion); __weak OWSBackupExportJob *weakSelf = self; - if (self.databaseFilePaths.count < 1) { + if (self.unsavedDatabaseItems.count < 1) { return NO; } - NSString *filePath = self.databaseFilePaths.lastObject; - [self.databaseFilePaths removeLastObject]; - // Database files are encrypted and can be safely stored unencrypted in the cloud. - // TODO: Security review. - [OWSBackupAPI saveEphemeralDatabaseFileToCloudWithFileUrl:[NSURL fileURLWithPath:filePath] + // Pop next item from queue, preserving ordering. + OWSBackupExportItem *item = self.unsavedDatabaseItems.firstObject; + [self.unsavedDatabaseItems removeObjectAtIndex:0]; + + OWSAssert(item.encryptedItem.filePath.length > 0); + + [OWSBackupAPI saveEphemeralDatabaseFileToCloudWithFileUrl:[NSURL fileURLWithPath:item.encryptedItem.filePath] success:^(NSString *recordName) { // Ensure that we continue to work off the main thread. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - OWSBackupExportJob *strongSelf = weakSelf; - if (!strongSelf) { - return; - } - strongSelf.databaseRecordMap[recordName] = [filePath lastPathComponent]; - [strongSelf saveNextFileToCloud:completion]; + item.recordName = recordName; + [weakSelf.savedDatabaseItems addObject:item]; + [weakSelf saveNextFileToCloud:completion]; }); } failure:^(NSError *error) { // Database files are critical so any error uploading them is unrecoverable. - DDLogVerbose(@"%@ error while saving file: %@", self.logTag, filePath); + DDLogVerbose(@"%@ error while saving file: %@", weakSelf.logTag, item.encryptedItem.filePath); completion(error); }]; return YES; } +// This method returns YES IFF "work was done and there might be more work to do". - (BOOL)saveNextAttachmentFileToCloud:(OWSBackupJobCompletion)completion { OWSAssert(completion); __weak OWSBackupExportJob *weakSelf = self; - if (self.attachmentFilePathMap.count < 1) { + if (self.unsavedAttachmentExports.count < 1) { return NO; } - NSString *attachmentId = self.attachmentFilePathMap.allKeys.lastObject; - NSString *attachmentFilePath = self.attachmentFilePathMap[attachmentId]; - [self.attachmentFilePathMap removeObjectForKey:attachmentId]; + // No need to preserve ordering of attachments. + OWSAttachmentExport *attachmentExport = self.unsavedAttachmentExports.lastObject; + [self.unsavedAttachmentExports removeLastObject]; // OWSAttachmentExport is used to lazily write an encrypted copy of the // attachment to disk. - OWSAttachmentExport *attachmentExport = [OWSAttachmentExport new]; - attachmentExport.delegate = self.delegate; - attachmentExport.jobTempDirPath = self.jobTempDirPath; - attachmentExport.attachmentId = attachmentId; - attachmentExport.attachmentFilePath = attachmentFilePath; + if (![attachmentExport prepareForUpload]) { + // Attachment files are non-critical so any error uploading them is recoverable. + [weakSelf saveNextFileToCloud:completion]; + return YES; + } + OWSAssert(attachmentExport.relativeFilePath.length > 0); + OWSAssert(attachmentExport.encryptedItem); - [OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentId + [OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentExport.attachmentId fileUrlBlock:^{ - [attachmentExport prepareForUpload]; - if (attachmentExport.tempFilePath.length < 1) { + if (attachmentExport.encryptedItem.filePath.length < 1) { DDLogError(@"%@ attachment export missing temp file path", self.logTag); return (NSURL *)nil; } @@ -854,7 +661,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe DDLogError(@"%@ attachment export missing relative file path", self.logTag); return (NSURL *)nil; } - return [NSURL fileURLWithPath:attachmentExport.tempFilePath]; + return [NSURL fileURLWithPath:attachmentExport.encryptedItem.filePath]; } success:^(NSString *recordName) { // Ensure that we continue to work off the main thread. @@ -863,10 +670,16 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe if (!strongSelf) { return; } - strongSelf.attachmentRecordMap[recordName] = attachmentExport.relativeFilePath; - DDLogVerbose(@"%@ exported attachment: %@ as %@", + + OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; + exportItem.encryptedItem = attachmentExport.encryptedItem; + exportItem.recordName = recordName; + exportItem.attachmentExport = attachmentExport; + [strongSelf.savedAttachmentItems addObject:exportItem]; + + DDLogVerbose(@"%@ saved attachment: %@ as %@", self.logTag, - attachmentFilePath, + attachmentExport.attachmentFilePath, attachmentExport.relativeFilePath); [strongSelf saveNextFileToCloud:completion]; }); @@ -878,6 +691,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [weakSelf saveNextFileToCloud:completion]; }); }]; + return YES; } @@ -885,17 +699,20 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe { OWSAssert(completion); - if (![self writeManifestFile]) { + OWSBackupEncryptedItem *_Nullable encryptedItem = [self writeManifestFile]; + if (!encryptedItem) { completion(OWSErrorWithCodeDescription(OWSErrorCodeExportBackupFailed, NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", @"Error indicating the a backup export could not export the user's data."))); return; } - OWSAssert(self.manifestFilePath); + + OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; + exportItem.encryptedItem = encryptedItem; __weak OWSBackupExportJob *weakSelf = self; - [OWSBackupAPI upsertManifestFileToCloudWithFileUrl:[NSURL fileURLWithPath:self.manifestFilePath] + [OWSBackupAPI upsertManifestFileToCloudWithFileUrl:[NSURL fileURLWithPath:encryptedItem.filePath] success:^(NSString *recordName) { // Ensure that we continue to work off the main thread. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ @@ -903,7 +720,9 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe if (!strongSelf) { return; } - strongSelf.manifestRecordName = recordName; + + exportItem.recordName = recordName; + strongSelf.manifestItem = exportItem; // All files have been saved to the cloud. completion(nil); @@ -915,24 +734,16 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe }]; } -- (BOOL)writeManifestFile +- (nullable OWSBackupEncryptedItem *)writeManifestFile { - OWSAssert(self.databaseRecordMap.count > 0); - OWSAssert(self.attachmentRecordMap); + OWSAssert(self.savedDatabaseItems.count > 0); + OWSAssert(self.savedAttachmentItems); OWSAssert(self.jobTempDirPath.length > 0); - - NSData *_Nullable databaseKeySpec = - [OWSBackupJob loadDatabaseKeySpecWithKeychainKey:kOWSBackup_ExportDatabaseKeySpec]; - if (databaseKeySpec.length < 1) { - OWSProdLogAndFail(@"%@ Could not load database keyspec for export.", self.logTag); - return nil; - } + OWSAssert(self.encryption); NSDictionary *json = @{ - kOWSBackup_ManifestKey_DatabaseFiles : self.databaseRecordMap, - kOWSBackup_ManifestKey_AttachmentFiles : self.attachmentRecordMap, - // JSON doesn't support byte arrays. - kOWSBackup_ManifestKey_DatabaseKeySpec : databaseKeySpec.base64EncodedString, + kOWSBackup_ManifestKey_DatabaseFiles : [self jsonForItems:self.savedDatabaseItems], + kOWSBackup_ManifestKey_AttachmentFiles : [self jsonForItems:self.savedAttachmentItems], }; DDLogVerbose(@"%@ json: %@", self.logTag, json); @@ -942,11 +753,30 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [NSJSONSerialization dataWithJSONObject:json options:NSJSONWritingPrettyPrinted error:&error]; if (!jsonData || error) { OWSProdLogAndFail(@"%@ error encoding manifest file: %@", self.logTag, error); - return NO; + return nil; } - self.manifestFilePath = - [OWSBackupJob encryptDataAsTempFile:jsonData jobTempDirPath:self.jobTempDirPath delegate:self.delegate]; - return self.manifestFilePath != nil; + return [self.encryption encryptDataAsTempFile:jsonData encryptionKey:self.delegate.backupEncryptionKey]; +} + +- (NSArray *> *)jsonForItems:(NSArray *)items +{ + NSMutableArray *result = [NSMutableArray new]; + for (OWSBackupExportItem *item in items) { + NSMutableDictionary *itemJson = [NSMutableDictionary new]; + OWSAssert(item.recordName.length > 0); + + itemJson[kOWSBackup_ManifestKey_RecordName] = item.recordName; + OWSAssert(item.encryptedItem.filePath.length > 0); + OWSAssert(item.encryptedItem.encryptionKey.length > 0); + itemJson[kOWSBackup_ManifestKey_EncryptionKey] = item.encryptedItem.encryptionKey.base64EncodedString; + if (item.attachmentExport) { + OWSAssert(item.attachmentExport.relativeFilePath.length > 0); + itemJson[kOWSBackup_ManifestKey_RelativeFilePath] = item.attachmentExport.relativeFilePath; + } + [result addObject:itemJson]; + } + + return result; } - (void)cleanUpCloud:(OWSBackupJobCompletion)completion @@ -964,14 +794,24 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe // records not involved in this backup export. NSMutableSet *activeRecordNames = [NSMutableSet new]; - OWSAssert(self.databaseRecordMap.count > 0); - [activeRecordNames addObjectsFromArray:self.databaseRecordMap.allKeys]; + OWSAssert(self.savedDatabaseItems.count > 0); + for (OWSBackupExportItem *item in self.savedDatabaseItems) { + OWSAssert(item.recordName.length > 0); + OWSAssert(![activeRecordNames containsObject:item.recordName]); + [activeRecordNames addObject:item.recordName]; + } + for (OWSBackupExportItem *item in self.savedAttachmentItems) { + OWSAssert(item.recordName.length > 0); + OWSAssert(![activeRecordNames containsObject:item.recordName]); + [activeRecordNames addObject:item.recordName]; + } + OWSAssert(self.manifestItem.recordName.length > 0); + OWSAssert(![activeRecordNames containsObject:self.manifestItem.recordName]); + [activeRecordNames addObject:self.manifestItem.recordName]; - OWSAssert(self.attachmentRecordMap); - [activeRecordNames addObjectsFromArray:self.attachmentRecordMap.allKeys]; - - OWSAssert(self.manifestRecordName.length > 0); - [activeRecordNames addObject:self.manifestRecordName]; + // TODO: If we implement "lazy restores" where attachments (etc.) are + // restored lazily, we need to include the record names for all + // records that haven't been restored yet. __weak OWSBackupExportJob *weakSelf = self; [OWSBackupAPI fetchAllRecordNamesWithSuccess:^(NSArray *recordNames) { diff --git a/Signal/src/util/OWSBackupImportJob.m b/Signal/src/util/OWSBackupImportJob.m index eae4ad2d90..6edc4f1203 100644 --- a/Signal/src/util/OWSBackupImportJob.m +++ b/Signal/src/util/OWSBackupImportJob.m @@ -3,6 +3,7 @@ // #import "OWSBackupImportJob.h" +#import "OWSBackupEncryption.h" #import "OWSDatabaseMigration.h" #import "OWSDatabaseMigrationRunner.h" #import "Signal-Swift.h" diff --git a/Signal/src/util/OWSBackupJob.h b/Signal/src/util/OWSBackupJob.h index ee5b4aa0b8..146e8ba7b9 100644 --- a/Signal/src/util/OWSBackupJob.h +++ b/Signal/src/util/OWSBackupJob.h @@ -6,10 +6,9 @@ NS_ASSUME_NONNULL_BEGIN extern NSString *const kOWSBackup_ManifestKey_DatabaseFiles; extern NSString *const kOWSBackup_ManifestKey_AttachmentFiles; -extern NSString *const kOWSBackup_ManifestKey_DatabaseKeySpec; - -extern NSString *const kOWSBackup_Snapshot_Collection; -extern NSString *const kOWSBackup_Snapshot_ValidKey; +extern NSString *const kOWSBackup_ManifestKey_RecordName; +extern NSString *const kOWSBackup_ManifestKey_EncryptionKey; +extern NSString *const kOWSBackup_ManifestKey_RelativeFilePath; typedef void (^OWSBackupJobBoolCompletion)(BOOL success); typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error); @@ -18,8 +17,7 @@ typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error); @protocol OWSBackupJobDelegate -// TODO: Use actual key. -- (nullable NSData *)backupKey; +- (nullable NSData *)backupEncryptionKey; // Either backupJobDidSucceed:... or backupJobDidFail:... will // be called exactly once on the main thread UNLESS: @@ -64,23 +62,12 @@ typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error); - (void)failWithError:(NSError *)error; - (void)updateProgressWithDescription:(nullable NSString *)description progress:(nullable NSNumber *)progress; - #pragma mark - Database KeySpec + (nullable NSData *)loadDatabaseKeySpecWithKeychainKey:(NSString *)keychainKey; + (BOOL)storeDatabaseKeySpec:(NSData *)data keychainKey:(NSString *)keychainKey; + (BOOL)generateRandomDatabaseKeySpecWithKeychainKey:(NSString *)keychainKey; -#pragma mark - Encryption - -+ (nullable NSString *)encryptFileAsTempFile:(NSString *)srcFilePath - jobTempDirPath:(NSString *)jobTempDirPath - delegate:(id)delegate; - -+ (nullable NSString *)encryptDataAsTempFile:(NSData *)data - jobTempDirPath:(NSString *)jobTempDirPath - delegate:(id)delegate; - @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackupJob.m b/Signal/src/util/OWSBackupJob.m index 84ba42b56c..d298979abd 100644 --- a/Signal/src/util/OWSBackupJob.m +++ b/Signal/src/util/OWSBackupJob.m @@ -3,6 +3,7 @@ // #import "OWSBackupJob.h" +#import "OWSBackupEncryption.h" #import "Signal-Swift.h" #import #import @@ -12,13 +13,12 @@ NS_ASSUME_NONNULL_BEGIN NSString *const kOWSBackup_ManifestKey_DatabaseFiles = @"database_files"; NSString *const kOWSBackup_ManifestKey_AttachmentFiles = @"attachment_files"; -NSString *const kOWSBackup_ManifestKey_DatabaseKeySpec = @"database_key_spec"; +NSString *const kOWSBackup_ManifestKey_RecordName = @"record_name"; +NSString *const kOWSBackup_ManifestKey_EncryptionKey = @"encryption_key"; +NSString *const kOWSBackup_ManifestKey_RelativeFilePath = @"relative_file_path"; NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; -NSString *const kOWSBackup_Snapshot_Collection = @"kOWSBackup_Snapshot_Collection"; -NSString *const kOWSBackup_Snapshot_ValidKey = @"kOWSBackup_Snapshot_ValidKey"; - @interface OWSBackupJob () @property (nonatomic, weak) id delegate; @@ -191,55 +191,6 @@ NSString *const kOWSBackup_Snapshot_ValidKey = @"kOWSBackup_Snapshot_ValidKey"; return [self storeDatabaseKeySpec:databaseKeySpec keychainKey:keychainKey]; } -#pragma mark - Encryption - -+ (nullable NSString *)encryptFileAsTempFile:(NSString *)srcFilePath - jobTempDirPath:(NSString *)jobTempDirPath - delegate:(id)delegate -{ - OWSAssert(srcFilePath.length > 0); - OWSAssert(jobTempDirPath.length > 0); - OWSAssert(delegate); - - // TODO: Encrypt the file using self.delegate.backupKey; - NSData *_Nullable backupKey = [delegate backupKey]; - OWSAssert(backupKey); - - NSString *dstFilePath = [jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; - NSFileManager *fileManager = [NSFileManager defaultManager]; - NSError *error; - BOOL success = [fileManager copyItemAtPath:srcFilePath toPath:dstFilePath error:&error]; - if (!success || error) { - OWSProdLogAndFail(@"%@ error writing encrypted file: %@", self.logTag, error); - return nil; - } - [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; - return dstFilePath; -} - -+ (nullable NSString *)encryptDataAsTempFile:(NSData *)data - jobTempDirPath:(NSString *)jobTempDirPath - delegate:(id)delegate -{ - OWSAssert(data); - OWSAssert(jobTempDirPath.length > 0); - OWSAssert(delegate); - - // TODO: Encrypt the file using self.delegate.backupKey; - NSData *_Nullable backupKey = [delegate backupKey]; - OWSAssert(backupKey); - - NSString *dstFilePath = [jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; - NSError *error; - BOOL success = [data writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; - if (!success || error) { - OWSProdLogAndFail(@"%@ error writing encrypted file: %@", self.logTag, error); - return nil; - } - [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; - return dstFilePath; -} - @end NS_ASSUME_NONNULL_END diff --git a/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto b/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto index 263ff798c2..e4ab31e8d9 100644 --- a/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto +++ b/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto @@ -238,3 +238,21 @@ message GroupDetails { optional bool active = 5 [default = true]; optional uint32 expireTimer = 6; } + +message BackupSnapshot +{ + message BackupEntity + { + enum Type { + UNKNOWN = 0; + MIGRATION = 1; + THREAD = 2; + INTERACTION = 3; + ATTACHMENT = 4; + } + optional Type type = 1; + optional bytes data = 2; + } + + repeated BackupEntity entity = 1; +} \ No newline at end of file diff --git a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h index 2307512a3a..59e6837d19 100644 --- a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h +++ b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h @@ -6,6 +6,10 @@ @class OWSSignalServiceProtosAttachmentPointer; @class OWSSignalServiceProtosAttachmentPointerBuilder; +@class OWSSignalServiceProtosBackupSnapshot; +@class OWSSignalServiceProtosBackupSnapshotBackupEntity; +@class OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder; +@class OWSSignalServiceProtosBackupSnapshotBuilder; @class OWSSignalServiceProtosCallMessage; @class OWSSignalServiceProtosCallMessageAnswer; @class OWSSignalServiceProtosCallMessageAnswerBuilder; @@ -168,6 +172,17 @@ typedef NS_ENUM(SInt32, OWSSignalServiceProtosGroupContextType) { BOOL OWSSignalServiceProtosGroupContextTypeIsValidValue(OWSSignalServiceProtosGroupContextType value); NSString *NSStringFromOWSSignalServiceProtosGroupContextType(OWSSignalServiceProtosGroupContextType value); +typedef NS_ENUM(SInt32, OWSSignalServiceProtosBackupSnapshotBackupEntityType) { + OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown = 0, + OWSSignalServiceProtosBackupSnapshotBackupEntityTypeMigration = 1, + OWSSignalServiceProtosBackupSnapshotBackupEntityTypeThread = 2, + OWSSignalServiceProtosBackupSnapshotBackupEntityTypeInteraction = 3, + OWSSignalServiceProtosBackupSnapshotBackupEntityTypeAttachment = 4, +}; + +BOOL OWSSignalServiceProtosBackupSnapshotBackupEntityTypeIsValidValue(OWSSignalServiceProtosBackupSnapshotBackupEntityType value); +NSString *NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(OWSSignalServiceProtosBackupSnapshotBackupEntityType value); + @interface OWSSignalServiceProtosOwssignalServiceProtosRoot : NSObject { } @@ -2224,5 +2239,115 @@ NSString *NSStringFromOWSSignalServiceProtosGroupContextType(OWSSignalServicePro - (OWSSignalServiceProtosGroupDetailsBuilder*) clearExpireTimer; @end +#define BackupSnapshot_entity @"entity" +@interface OWSSignalServiceProtosBackupSnapshot : PBGeneratedMessage { +@private + NSMutableArray * entityArray; +} +@property (readonly, strong) NSArray * entity; +- (OWSSignalServiceProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index; + ++ (instancetype) defaultInstance; +- (instancetype) defaultInstance; + +- (BOOL) isInitialized; +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output; +- (OWSSignalServiceProtosBackupSnapshotBuilder*) builder; ++ (OWSSignalServiceProtosBackupSnapshotBuilder*) builder; ++ (OWSSignalServiceProtosBackupSnapshotBuilder*) builderWithPrototype:(OWSSignalServiceProtosBackupSnapshot*) prototype; +- (OWSSignalServiceProtosBackupSnapshotBuilder*) toBuilder; + ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromData:(NSData*) data; ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input; ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input; ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; +@end + +#define BackupEntity_type @"type" +#define BackupEntity_data @"data" +@interface OWSSignalServiceProtosBackupSnapshotBackupEntity : PBGeneratedMessage { +@private + BOOL hasData_:1; + BOOL hasType_:1; + NSData* data; + OWSSignalServiceProtosBackupSnapshotBackupEntityType type; +} +- (BOOL) hasType; +- (BOOL) hasData; +@property (readonly) OWSSignalServiceProtosBackupSnapshotBackupEntityType type; +@property (readonly, strong) NSData* data; + ++ (instancetype) defaultInstance; +- (instancetype) defaultInstance; + +- (BOOL) isInitialized; +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builder; ++ (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builder; ++ (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builderWithPrototype:(OWSSignalServiceProtosBackupSnapshotBackupEntity*) prototype; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) toBuilder; + ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data; ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input; ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input; ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; +@end + +@interface OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder : PBGeneratedMessageBuilder { +@private + OWSSignalServiceProtosBackupSnapshotBackupEntity* resultBackupEntity; +} + +- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) defaultInstance; + +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clear; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clone; + +- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) build; +- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) buildPartial; + +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFrom:(OWSSignalServiceProtosBackupSnapshotBackupEntity*) other; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; + +- (BOOL) hasType; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityType) type; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setType:(OWSSignalServiceProtosBackupSnapshotBackupEntityType) value; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearType; + +- (BOOL) hasData; +- (NSData*) data; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setData:(NSData*) value; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearData; +@end + +@interface OWSSignalServiceProtosBackupSnapshotBuilder : PBGeneratedMessageBuilder { +@private + OWSSignalServiceProtosBackupSnapshot* resultBackupSnapshot; +} + +- (OWSSignalServiceProtosBackupSnapshot*) defaultInstance; + +- (OWSSignalServiceProtosBackupSnapshotBuilder*) clear; +- (OWSSignalServiceProtosBackupSnapshotBuilder*) clone; + +- (OWSSignalServiceProtosBackupSnapshot*) build; +- (OWSSignalServiceProtosBackupSnapshot*) buildPartial; + +- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFrom:(OWSSignalServiceProtosBackupSnapshot*) other; +- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input; +- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; + +- (NSMutableArray *)entity; +- (OWSSignalServiceProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index; +- (OWSSignalServiceProtosBackupSnapshotBuilder *)addEntity:(OWSSignalServiceProtosBackupSnapshotBackupEntity*)value; +- (OWSSignalServiceProtosBackupSnapshotBuilder *)setEntityArray:(NSArray *)array; +- (OWSSignalServiceProtosBackupSnapshotBuilder *)clearEntity; +@end + // @@protoc_insertion_point(global_scope) diff --git a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m index 9b81f29081..17911e653b 100644 --- a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m +++ b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m @@ -9705,5 +9705,515 @@ static OWSSignalServiceProtosGroupDetailsAvatar* defaultOWSSignalServiceProtosGr } @end +@interface OWSSignalServiceProtosBackupSnapshot () +@property (strong) NSMutableArray * entityArray; +@end + +@implementation OWSSignalServiceProtosBackupSnapshot + +@synthesize entityArray; +@dynamic entity; +- (instancetype) init { + if ((self = [super init])) { + } + return self; +} +static OWSSignalServiceProtosBackupSnapshot* defaultOWSSignalServiceProtosBackupSnapshotInstance = nil; ++ (void) initialize { + if (self == [OWSSignalServiceProtosBackupSnapshot class]) { + defaultOWSSignalServiceProtosBackupSnapshotInstance = [[OWSSignalServiceProtosBackupSnapshot alloc] init]; + } +} ++ (instancetype) defaultInstance { + return defaultOWSSignalServiceProtosBackupSnapshotInstance; +} +- (instancetype) defaultInstance { + return defaultOWSSignalServiceProtosBackupSnapshotInstance; +} +- (NSArray *)entity { + return entityArray; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index { + return [entityArray objectAtIndex:index]; +} +- (BOOL) isInitialized { + return YES; +} +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output { + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignalServiceProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + [output writeMessage:1 value:element]; + }]; + [self.unknownFields writeToCodedOutputStream:output]; +} +- (SInt32) serializedSize { + __block SInt32 size_ = memoizedSerializedSize; + if (size_ != -1) { + return size_; + } + + size_ = 0; + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignalServiceProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + size_ += computeMessageSize(1, element); + }]; + size_ += self.unknownFields.serializedSize; + memoizedSerializedSize = size_; + return size_; +} ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromData:(NSData*) data { + return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromData:data] build]; +} ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromData:data extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input { + return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromInputStream:input] build]; +} ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input { + return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromCodedInputStream:input] build]; +} ++ (OWSSignalServiceProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromCodedInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignalServiceProtosBackupSnapshotBuilder*) builder { + return [[OWSSignalServiceProtosBackupSnapshotBuilder alloc] init]; +} ++ (OWSSignalServiceProtosBackupSnapshotBuilder*) builderWithPrototype:(OWSSignalServiceProtosBackupSnapshot*) prototype { + return [[OWSSignalServiceProtosBackupSnapshot builder] mergeFrom:prototype]; +} +- (OWSSignalServiceProtosBackupSnapshotBuilder*) builder { + return [OWSSignalServiceProtosBackupSnapshot builder]; +} +- (OWSSignalServiceProtosBackupSnapshotBuilder*) toBuilder { + return [OWSSignalServiceProtosBackupSnapshot builderWithPrototype:self]; +} +- (void) writeDescriptionTo:(NSMutableString*) output withIndent:(NSString*) indent { + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignalServiceProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + [output appendFormat:@"%@%@ {\n", indent, @"entity"]; + [element writeDescriptionTo:output + withIndent:[NSString stringWithFormat:@"%@ ", indent]]; + [output appendFormat:@"%@}\n", indent]; + }]; + [self.unknownFields writeDescriptionTo:output withIndent:indent]; +} +- (void) storeInDictionary:(NSMutableDictionary *)dictionary { + for (OWSSignalServiceProtosBackupSnapshotBackupEntity* element in self.entityArray) { + NSMutableDictionary *elementDictionary = [NSMutableDictionary dictionary]; + [element storeInDictionary:elementDictionary]; + [dictionary setObject:[NSDictionary dictionaryWithDictionary:elementDictionary] forKey:@"entity"]; + } + [self.unknownFields storeInDictionary:dictionary]; +} +- (BOOL) isEqual:(id)other { + if (other == self) { + return YES; + } + if (![other isKindOfClass:[OWSSignalServiceProtosBackupSnapshot class]]) { + return NO; + } + OWSSignalServiceProtosBackupSnapshot *otherMessage = other; + return + [self.entityArray isEqualToArray:otherMessage.entityArray] && + (self.unknownFields == otherMessage.unknownFields || (self.unknownFields != nil && [self.unknownFields isEqual:otherMessage.unknownFields])); +} +- (NSUInteger) hash { + __block NSUInteger hashCode = 7; + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignalServiceProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + hashCode = hashCode * 31 + [element hash]; + }]; + hashCode = hashCode * 31 + [self.unknownFields hash]; + return hashCode; +} +@end + +@interface OWSSignalServiceProtosBackupSnapshotBackupEntity () +@property OWSSignalServiceProtosBackupSnapshotBackupEntityType type; +@property (strong) NSData* data; +@end + +@implementation OWSSignalServiceProtosBackupSnapshotBackupEntity + +- (BOOL) hasType { + return !!hasType_; +} +- (void) setHasType:(BOOL) _value_ { + hasType_ = !!_value_; +} +@synthesize type; +- (BOOL) hasData { + return !!hasData_; +} +- (void) setHasData:(BOOL) _value_ { + hasData_ = !!_value_; +} +@synthesize data; +- (instancetype) init { + if ((self = [super init])) { + self.type = OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown; + self.data = [NSData data]; + } + return self; +} +static OWSSignalServiceProtosBackupSnapshotBackupEntity* defaultOWSSignalServiceProtosBackupSnapshotBackupEntityInstance = nil; ++ (void) initialize { + if (self == [OWSSignalServiceProtosBackupSnapshotBackupEntity class]) { + defaultOWSSignalServiceProtosBackupSnapshotBackupEntityInstance = [[OWSSignalServiceProtosBackupSnapshotBackupEntity alloc] init]; + } +} ++ (instancetype) defaultInstance { + return defaultOWSSignalServiceProtosBackupSnapshotBackupEntityInstance; +} +- (instancetype) defaultInstance { + return defaultOWSSignalServiceProtosBackupSnapshotBackupEntityInstance; +} +- (BOOL) isInitialized { + return YES; +} +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output { + if (self.hasType) { + [output writeEnum:1 value:self.type]; + } + if (self.hasData) { + [output writeData:2 value:self.data]; + } + [self.unknownFields writeToCodedOutputStream:output]; +} +- (SInt32) serializedSize { + __block SInt32 size_ = memoizedSerializedSize; + if (size_ != -1) { + return size_; + } + + size_ = 0; + if (self.hasType) { + size_ += computeEnumSize(1, self.type); + } + if (self.hasData) { + size_ += computeDataSize(2, self.data); + } + size_ += self.unknownFields.serializedSize; + memoizedSerializedSize = size_; + return size_; +} ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data { + return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromData:data] build]; +} ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromData:data extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input { + return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromInputStream:input] build]; +} ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input { + return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromCodedInputStream:input] build]; +} ++ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromCodedInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builder { + return [[OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder alloc] init]; +} ++ (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builderWithPrototype:(OWSSignalServiceProtosBackupSnapshotBackupEntity*) prototype { + return [[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFrom:prototype]; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builder { + return [OWSSignalServiceProtosBackupSnapshotBackupEntity builder]; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) toBuilder { + return [OWSSignalServiceProtosBackupSnapshotBackupEntity builderWithPrototype:self]; +} +- (void) writeDescriptionTo:(NSMutableString*) output withIndent:(NSString*) indent { + if (self.hasType) { + [output appendFormat:@"%@%@: %@\n", indent, @"type", NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(self.type)]; + } + if (self.hasData) { + [output appendFormat:@"%@%@: %@\n", indent, @"data", self.data]; + } + [self.unknownFields writeDescriptionTo:output withIndent:indent]; +} +- (void) storeInDictionary:(NSMutableDictionary *)dictionary { + if (self.hasType) { + [dictionary setObject: @(self.type) forKey: @"type"]; + } + if (self.hasData) { + [dictionary setObject: self.data forKey: @"data"]; + } + [self.unknownFields storeInDictionary:dictionary]; +} +- (BOOL) isEqual:(id)other { + if (other == self) { + return YES; + } + if (![other isKindOfClass:[OWSSignalServiceProtosBackupSnapshotBackupEntity class]]) { + return NO; + } + OWSSignalServiceProtosBackupSnapshotBackupEntity *otherMessage = other; + return + self.hasType == otherMessage.hasType && + (!self.hasType || self.type == otherMessage.type) && + self.hasData == otherMessage.hasData && + (!self.hasData || [self.data isEqual:otherMessage.data]) && + (self.unknownFields == otherMessage.unknownFields || (self.unknownFields != nil && [self.unknownFields isEqual:otherMessage.unknownFields])); +} +- (NSUInteger) hash { + __block NSUInteger hashCode = 7; + if (self.hasType) { + hashCode = hashCode * 31 + self.type; + } + if (self.hasData) { + hashCode = hashCode * 31 + [self.data hash]; + } + hashCode = hashCode * 31 + [self.unknownFields hash]; + return hashCode; +} +@end + +BOOL OWSSignalServiceProtosBackupSnapshotBackupEntityTypeIsValidValue(OWSSignalServiceProtosBackupSnapshotBackupEntityType value) { + switch (value) { + case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown: + case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeMigration: + case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeThread: + case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeInteraction: + case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeAttachment: + return YES; + default: + return NO; + } +} +NSString *NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(OWSSignalServiceProtosBackupSnapshotBackupEntityType value) { + switch (value) { + case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown: + return @"OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown"; + case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeMigration: + return @"OWSSignalServiceProtosBackupSnapshotBackupEntityTypeMigration"; + case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeThread: + return @"OWSSignalServiceProtosBackupSnapshotBackupEntityTypeThread"; + case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeInteraction: + return @"OWSSignalServiceProtosBackupSnapshotBackupEntityTypeInteraction"; + case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeAttachment: + return @"OWSSignalServiceProtosBackupSnapshotBackupEntityTypeAttachment"; + default: + return nil; + } +} + +@interface OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder() +@property (strong) OWSSignalServiceProtosBackupSnapshotBackupEntity* resultBackupEntity; +@end + +@implementation OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder +@synthesize resultBackupEntity; +- (instancetype) init { + if ((self = [super init])) { + self.resultBackupEntity = [[OWSSignalServiceProtosBackupSnapshotBackupEntity alloc] init]; + } + return self; +} +- (PBGeneratedMessage*) internalGetResult { + return resultBackupEntity; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clear { + self.resultBackupEntity = [[OWSSignalServiceProtosBackupSnapshotBackupEntity alloc] init]; + return self; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clone { + return [OWSSignalServiceProtosBackupSnapshotBackupEntity builderWithPrototype:resultBackupEntity]; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) defaultInstance { + return [OWSSignalServiceProtosBackupSnapshotBackupEntity defaultInstance]; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) build { + [self checkInitialized]; + return [self buildPartial]; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) buildPartial { + OWSSignalServiceProtosBackupSnapshotBackupEntity* returnMe = resultBackupEntity; + self.resultBackupEntity = nil; + return returnMe; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFrom:(OWSSignalServiceProtosBackupSnapshotBackupEntity*) other { + if (other == [OWSSignalServiceProtosBackupSnapshotBackupEntity defaultInstance]) { + return self; + } + if (other.hasType) { + [self setType:other.type]; + } + if (other.hasData) { + [self setData:other.data]; + } + [self mergeUnknownFields:other.unknownFields]; + return self; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input { + return [self mergeFromCodedInputStream:input extensionRegistry:[PBExtensionRegistry emptyRegistry]]; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + PBUnknownFieldSetBuilder* unknownFields = [PBUnknownFieldSet builderWithUnknownFields:self.unknownFields]; + while (YES) { + SInt32 tag = [input readTag]; + switch (tag) { + case 0: + [self setUnknownFields:[unknownFields build]]; + return self; + default: { + if (![self parseUnknownField:input unknownFields:unknownFields extensionRegistry:extensionRegistry tag:tag]) { + [self setUnknownFields:[unknownFields build]]; + return self; + } + break; + } + case 8: { + OWSSignalServiceProtosBackupSnapshotBackupEntityType value = (OWSSignalServiceProtosBackupSnapshotBackupEntityType)[input readEnum]; + if (OWSSignalServiceProtosBackupSnapshotBackupEntityTypeIsValidValue(value)) { + [self setType:value]; + } else { + [unknownFields mergeVarintField:1 value:value]; + } + break; + } + case 18: { + [self setData:[input readData]]; + break; + } + } + } +} +- (BOOL) hasType { + return resultBackupEntity.hasType; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityType) type { + return resultBackupEntity.type; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setType:(OWSSignalServiceProtosBackupSnapshotBackupEntityType) value { + resultBackupEntity.hasType = YES; + resultBackupEntity.type = value; + return self; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearType { + resultBackupEntity.hasType = NO; + resultBackupEntity.type = OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown; + return self; +} +- (BOOL) hasData { + return resultBackupEntity.hasData; +} +- (NSData*) data { + return resultBackupEntity.data; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setData:(NSData*) value { + resultBackupEntity.hasData = YES; + resultBackupEntity.data = value; + return self; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearData { + resultBackupEntity.hasData = NO; + resultBackupEntity.data = [NSData data]; + return self; +} +@end + +@interface OWSSignalServiceProtosBackupSnapshotBuilder() +@property (strong) OWSSignalServiceProtosBackupSnapshot* resultBackupSnapshot; +@end + +@implementation OWSSignalServiceProtosBackupSnapshotBuilder +@synthesize resultBackupSnapshot; +- (instancetype) init { + if ((self = [super init])) { + self.resultBackupSnapshot = [[OWSSignalServiceProtosBackupSnapshot alloc] init]; + } + return self; +} +- (PBGeneratedMessage*) internalGetResult { + return resultBackupSnapshot; +} +- (OWSSignalServiceProtosBackupSnapshotBuilder*) clear { + self.resultBackupSnapshot = [[OWSSignalServiceProtosBackupSnapshot alloc] init]; + return self; +} +- (OWSSignalServiceProtosBackupSnapshotBuilder*) clone { + return [OWSSignalServiceProtosBackupSnapshot builderWithPrototype:resultBackupSnapshot]; +} +- (OWSSignalServiceProtosBackupSnapshot*) defaultInstance { + return [OWSSignalServiceProtosBackupSnapshot defaultInstance]; +} +- (OWSSignalServiceProtosBackupSnapshot*) build { + [self checkInitialized]; + return [self buildPartial]; +} +- (OWSSignalServiceProtosBackupSnapshot*) buildPartial { + OWSSignalServiceProtosBackupSnapshot* returnMe = resultBackupSnapshot; + self.resultBackupSnapshot = nil; + return returnMe; +} +- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFrom:(OWSSignalServiceProtosBackupSnapshot*) other { + if (other == [OWSSignalServiceProtosBackupSnapshot defaultInstance]) { + return self; + } + if (other.entityArray.count > 0) { + if (resultBackupSnapshot.entityArray == nil) { + resultBackupSnapshot.entityArray = [[NSMutableArray alloc] initWithArray:other.entityArray]; + } else { + [resultBackupSnapshot.entityArray addObjectsFromArray:other.entityArray]; + } + } + [self mergeUnknownFields:other.unknownFields]; + return self; +} +- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input { + return [self mergeFromCodedInputStream:input extensionRegistry:[PBExtensionRegistry emptyRegistry]]; +} +- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + PBUnknownFieldSetBuilder* unknownFields = [PBUnknownFieldSet builderWithUnknownFields:self.unknownFields]; + while (YES) { + SInt32 tag = [input readTag]; + switch (tag) { + case 0: + [self setUnknownFields:[unknownFields build]]; + return self; + default: { + if (![self parseUnknownField:input unknownFields:unknownFields extensionRegistry:extensionRegistry tag:tag]) { + [self setUnknownFields:[unknownFields build]]; + return self; + } + break; + } + case 10: { + OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder* subBuilder = [OWSSignalServiceProtosBackupSnapshotBackupEntity builder]; + [input readMessage:subBuilder extensionRegistry:extensionRegistry]; + [self addEntity:[subBuilder buildPartial]]; + break; + } + } + } +} +- (NSMutableArray *)entity { + return resultBackupSnapshot.entityArray; +} +- (OWSSignalServiceProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index { + return [resultBackupSnapshot entityAtIndex:index]; +} +- (OWSSignalServiceProtosBackupSnapshotBuilder *)addEntity:(OWSSignalServiceProtosBackupSnapshotBackupEntity*)value { + if (resultBackupSnapshot.entityArray == nil) { + resultBackupSnapshot.entityArray = [[NSMutableArray alloc]init]; + } + [resultBackupSnapshot.entityArray addObject:value]; + return self; +} +- (OWSSignalServiceProtosBackupSnapshotBuilder *)setEntityArray:(NSArray *)array { + resultBackupSnapshot.entityArray = [[NSMutableArray alloc]initWithArray:array]; + return self; +} +- (OWSSignalServiceProtosBackupSnapshotBuilder *)clearEntity { + resultBackupSnapshot.entityArray = nil; + return self; +} +@end + // @@protoc_insertion_point(global_scope) From 0c81d5656f4b38972080f3da3012ff0ac55f383a Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Fri, 16 Mar 2018 13:55:20 -0300 Subject: [PATCH 03/13] Rework database snapshot representation, encryption, etc. --- Signal/src/util/OWSBackupEncryption.h | 12 +- Signal/src/util/OWSBackupEncryption.m | 118 +++- Signal/src/util/OWSBackupExportJob.m | 99 ++-- Signal/src/util/OWSBackupImportJob.m | 502 +++++++++--------- Signal/src/util/OWSBackupJob.h | 7 +- Signal/src/util/OWSBackupJob.m | 46 +- SignalServiceKit/protobuf/Makefile | 6 +- .../protobuf/OWSSignalServiceProtos.proto | 2 +- .../src/Messages/OWSSignalServiceProtos.pb.h | 18 +- .../src/Messages/OWSSignalServiceProtos.pb.m | 64 +-- SignalServiceKit/src/Storage/OWSStorage.m | 8 - 11 files changed, 470 insertions(+), 412 deletions(-) diff --git a/Signal/src/util/OWSBackupEncryption.h b/Signal/src/util/OWSBackupEncryption.h index 5729f59979..38a8a12814 100644 --- a/Signal/src/util/OWSBackupEncryption.h +++ b/Signal/src/util/OWSBackupEncryption.h @@ -33,10 +33,20 @@ NS_ASSUME_NONNULL_BEGIN #pragma mark - Decrypt -- (nullable NSString *)decryptFileAsTempFile:(NSString *)srcFilePath encryptionKey:(NSData *)encryptionKey; +- (BOOL)decryptFileAsFile:(NSString *)srcFilePath + dstFilePath:(NSString *)dstFilePath + encryptionKey:(NSData *)encryptionKey; + +- (nullable NSData *)decryptFileAsData:(NSString *)srcFilePath encryptionKey:(NSData *)encryptionKey; - (nullable NSData *)decryptDataAsData:(NSData *)srcData encryptionKey:(NSData *)encryptionKey; +#pragma mark - Compression + +- (nullable NSData *)compressData:(NSData *)srcData; + +- (nullable NSData *)decompressData:(NSData *)srcData uncompressedDataLength:(NSUInteger)uncompressedDataLength; + @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackupEncryption.m b/Signal/src/util/OWSBackupEncryption.m index 8b9ecd051f..c618a7e3e6 100644 --- a/Signal/src/util/OWSBackupEncryption.m +++ b/Signal/src/util/OWSBackupEncryption.m @@ -6,6 +6,8 @@ #import #import +@import Compression; + NS_ASSUME_NONNULL_BEGIN // TODO: @@ -95,11 +97,41 @@ static const NSUInteger kOWSBackupKeyLength = 32; #pragma mark - Decrypt -- (nullable NSString *)decryptFileAsTempFile:(NSString *)srcFilePath encryptionKey:(NSData *)encryptionKey +- (BOOL)decryptFileAsFile:(NSString *)srcFilePath + dstFilePath:(NSString *)dstFilePath + encryptionKey:(NSData *)encryptionKey { OWSAssert(srcFilePath.length > 0); OWSAssert(encryptionKey.length > 0); + // TODO: Decrypt the file without loading it into memory. + NSData *data = [self decryptFileAsData:srcFilePath encryptionKey:encryptionKey]; + + if (!data) { + return NO; + } + + NSError *error; + BOOL success = [data writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; + if (!success || error) { + OWSProdLogAndFail(@"%@ error writing decrypted data: %@", self.logTag, error); + return NO; + } + [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; + + return YES; +} + +- (nullable NSData *)decryptFileAsData:(NSString *)srcFilePath encryptionKey:(NSData *)encryptionKey +{ + OWSAssert(srcFilePath.length > 0); + OWSAssert(encryptionKey.length > 0); + + if (![NSFileManager.defaultManager fileExistsAtPath:srcFilePath]) { + DDLogError(@"%@ missing downloaded file.", self.logTag); + return nil; + } + // TODO: Decrypt the file without loading it into memory. NSData *_Nullable srcData = [NSData dataWithContentsOfFile:srcFilePath]; if (!srcData) { @@ -108,20 +140,7 @@ static const NSUInteger kOWSBackupKeyLength = 32; } NSData *_Nullable dstData = [self decryptDataAsData:srcData encryptionKey:encryptionKey]; - if (!dstData) { - return nil; - } - - NSString *dstFilePath = [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; - NSError *error; - BOOL success = [dstData writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; - if (!success || error) { - OWSProdLogAndFail(@"%@ error writing decrypted data: %@", self.logTag, error); - return nil; - } - [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; - - return dstFilePath; + return dstData; } - (nullable NSData *)decryptDataAsData:(NSData *)srcData encryptionKey:(NSData *)encryptionKey @@ -134,6 +153,75 @@ static const NSUInteger kOWSBackupKeyLength = 32; return srcData; } +#pragma mark - Compression + +- (nullable NSData *)compressData:(NSData *)srcData +{ + OWSAssert(srcData); + + if (!srcData) { + OWSProdLogAndFail(@"%@ missing unencrypted data.", self.logTag); + return nil; + } + + size_t srcLength = [srcData length]; + const uint8_t *srcBuffer = (const uint8_t *)[srcData bytes]; + if (!srcBuffer) { + return nil; + } + // This assumes that dst will always be smaller than src. + // + // We slightly pad the buffer size to account for the worst case. + uint8_t *dstBuffer = malloc(sizeof(uint8_t) * srcLength) + 64 * 1024; + if (!dstBuffer) { + return nil; + } + // TODO: Should we use COMPRESSION_LZFSE? + size_t dstLength = compression_encode_buffer(dstBuffer, srcLength, srcBuffer, srcLength, NULL, COMPRESSION_LZFSE); + NSData *compressedData = [NSData dataWithBytes:dstBuffer length:dstLength]; + DDLogVerbose(@"%@ compressed %zd -> %zd = %0.2f", + self.logTag, + srcLength, + dstLength, + (srcLength > 0 ? (dstLength / (CGFloat)srcLength) : 0)); + free(dstBuffer); + + return compressedData; +} + +- (nullable NSData *)decompressData:(NSData *)srcData uncompressedDataLength:(NSUInteger)uncompressedDataLength +{ + OWSAssert(srcData); + + if (!srcData) { + OWSProdLogAndFail(@"%@ missing unencrypted data.", self.logTag); + return nil; + } + + size_t srcLength = [srcData length]; + const uint8_t *srcBuffer = (const uint8_t *)[srcData bytes]; + if (!srcBuffer) { + return nil; + } + // We pad the buffer to be defensive. + uint8_t *dstBuffer = malloc(sizeof(uint8_t) * (uncompressedDataLength + 1024)); + if (!dstBuffer) { + return nil; + } + // TODO: Should we use COMPRESSION_LZFSE? + size_t dstLength = compression_decode_buffer(dstBuffer, srcLength, srcBuffer, srcLength, NULL, COMPRESSION_LZFSE); + NSData *decompressedData = [NSData dataWithBytes:dstBuffer length:dstLength]; + OWSAssert(decompressedData.length == uncompressedDataLength); + DDLogVerbose(@"%@ decompressed %zd -> %zd = %0.2f", + self.logTag, + srcLength, + dstLength, + (dstLength > 0 ? (srcLength / (CGFloat)dstLength) : 0)); + free(dstBuffer); + + return decompressedData; +} + @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index b832bae455..49c364df3b 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -3,13 +3,13 @@ // #import "OWSBackupExportJob.h" +#import "OWSBackupEncryption.h" #import "OWSDatabaseMigration.h" #import "OWSSignalServiceProtos.pb.h" #import "Signal-Swift.h" #import #import #import -#import #import #import #import @@ -18,11 +18,6 @@ #import #import -@import Compression; -#import "OWSBackupEncryption.h" - -// TODO: - NS_ASSUME_NONNULL_BEGIN @class OWSAttachmentExport; @@ -40,6 +35,9 @@ NS_ASSUME_NONNULL_BEGIN // This property is optional and is only used for attachments. @property (nonatomic, nullable) OWSAttachmentExport *attachmentExport; +// This property is optional. +@property (nonatomic, nullable) NSNumber *uncompressedDataLength; + - (instancetype)init NS_UNAVAILABLE; @end @@ -69,7 +67,7 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic) OWSBackupEncryption *encryption; -@property (nonatomic) NSMutableArray *encryptedItems; +@property (nonatomic) NSMutableArray *exportItems; @property (nonatomic, nullable) OWSSignalServiceProtosBackupSnapshotBuilder *backupSnapshotBuilder; @@ -93,7 +91,7 @@ NS_ASSUME_NONNULL_BEGIN OWSAssert(encryption); - self.encryptedItems = [NSMutableArray new]; + self.exportItems = [NSMutableArray new]; self.encryption = encryption; return self; @@ -117,7 +115,7 @@ NS_ASSUME_NONNULL_BEGIN OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder *entityBuilder = [OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder new]; [entityBuilder setType:entityType]; - [entityBuilder setData:data]; + [entityBuilder setEntityData:data]; [self.backupSnapshotBuilder addEntity:[entityBuilder build]]; @@ -140,6 +138,7 @@ NS_ASSUME_NONNULL_BEGIN } NSData *_Nullable uncompressedData = [self.backupSnapshotBuilder build].data; + NSUInteger uncompressedDataLength = uncompressedData.length; self.backupSnapshotBuilder = nil; self.cachedItemCount = 0; if (!uncompressedData) { @@ -147,31 +146,18 @@ NS_ASSUME_NONNULL_BEGIN return NO; } - size_t srcLength = [uncompressedData length]; - const uint8_t *srcBuffer = (const uint8_t *)[uncompressedData bytes]; - if (!srcBuffer) { - return NO; - } - // This assumes that dst will always be smaller than src. - uint8_t *dstBuffer = malloc(sizeof(uint8_t) * srcLength); - if (!dstBuffer) { - return NO; - } - // TODO: Should we use COMPRESSION_LZFSE? - size_t dstLength = compression_encode_buffer(dstBuffer, srcLength, srcBuffer, srcLength, NULL, COMPRESSION_LZFSE); - NSData *compressedData = [NSData dataWithBytes:dstBuffer length:dstLength]; - DDLogVerbose(@"%@ compressed %zd -> %zd = %0.2f", - self.logTag, - srcLength, - dstLength, - (dstLength > srcLength ? (dstLength / (CGFloat)srcLength) : 0)); - free(dstBuffer); + NSData *compressedData = [self.encryption compressData:uncompressedData]; OWSBackupEncryptedItem *_Nullable encryptedItem = [self.encryption encryptDataAsTempFile:compressedData]; if (!encryptedItem) { OWSProdLogAndFail(@"%@ couldn't encrypt database snapshot.", self.logTag); return NO; } + + OWSBackupExportItem *exportItem = [[OWSBackupExportItem alloc] initWithEncryptedItem:encryptedItem]; + exportItem.uncompressedDataLength = @(uncompressedDataLength); + [self.exportItems addObject:exportItem]; + return YES; } @@ -519,8 +505,7 @@ NS_ASSUME_NONNULL_BEGIN return NO; } - self.unsavedDatabaseItems = - [[self exportItemsForEncryptedItems:exportStream.encryptedItems label:@"database snapshots"] mutableCopy]; + self.unsavedDatabaseItems = [exportStream.exportItems mutableCopy]; // TODO: Should we do a database checkpoint? @@ -533,26 +518,6 @@ NS_ASSUME_NONNULL_BEGIN return YES; } -- (NSArray *)exportItemsForEncryptedItems:(NSArray *)encryptedItems - label:(NSString *)label -{ - OWSAssert(encryptedItems); - OWSAssert(label.length); - - NSMutableArray *exportItems = [NSMutableArray new]; - unsigned long long totalFileSize = 0; - for (OWSBackupEncryptedItem *encryptedItem in encryptedItems) { - OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; - exportItem.encryptedItem = encryptedItem; - [exportItems addObject:exportItem]; - - totalFileSize += [OWSFileSystem fileSizeOfPath:encryptedItem.filePath].unsignedLongLongValue; - } - DDLogInfo(@"%@ exported %@: count: %zd, bytes: %llu.", self.logTag, label, exportItems.count, totalFileSize); - - return exportItems; -} - - (void)saveToCloud:(OWSBackupJobCompletion)completion { OWSAssert(completion); @@ -562,6 +527,29 @@ NS_ASSUME_NONNULL_BEGIN self.savedDatabaseItems = [NSMutableArray new]; self.savedAttachmentItems = [NSMutableArray new]; + { + unsigned long long totalFileSize = 0; + for (OWSBackupExportItem *item in self.unsavedDatabaseItems) { + totalFileSize += [OWSFileSystem fileSizeOfPath:item.encryptedItem.filePath].unsignedLongLongValue; + } + DDLogInfo(@"%@ exporting %@: count: %zd, bytes: %llu.", + self.logTag, + @"database items", + self.unsavedDatabaseItems.count, + totalFileSize); + } + { + unsigned long long totalFileSize = 0; + for (OWSAttachmentExport *attachmentExport in self.unsavedAttachmentExports) { + totalFileSize += [OWSFileSystem fileSizeOfPath:attachmentExport.attachmentFilePath].unsignedLongLongValue; + } + DDLogInfo(@"%@ exporting %@: count: %zd, bytes: %llu.", + self.logTag, + @"attachment items", + self.unsavedAttachmentExports.count, + totalFileSize); + } + [self saveNextFileToCloud:completion]; } @@ -773,6 +761,9 @@ NS_ASSUME_NONNULL_BEGIN OWSAssert(item.attachmentExport.relativeFilePath.length > 0); itemJson[kOWSBackup_ManifestKey_RelativeFilePath] = item.attachmentExport.relativeFilePath; } + if (item.uncompressedDataLength) { + itemJson[kOWSBackup_ManifestKey_DataSize] = item.uncompressedDataLength; + } [result addObject:itemJson]; } @@ -852,8 +843,12 @@ NS_ASSUME_NONNULL_BEGIN if (obsoleteRecordNames.count < 1) { // No more records to delete; cleanup is complete. - completion(nil); - return; + return completion(nil); + } + + if (self.isComplete) { + // Job was aborted. + return completion(nil); } CGFloat progress = (obsoleteRecordNames.count / (CGFloat)(obsoleteRecordNames.count + deletedCount)); diff --git a/Signal/src/util/OWSBackupImportJob.m b/Signal/src/util/OWSBackupImportJob.m index 6edc4f1203..a792d7f4f8 100644 --- a/Signal/src/util/OWSBackupImportJob.m +++ b/Signal/src/util/OWSBackupImportJob.m @@ -9,7 +9,6 @@ #import "Signal-Swift.h" #import #import -#import #import #import #import @@ -21,20 +20,36 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe #pragma mark - +@interface OWSBackupImportItem : NSObject + +@property (nonatomic) NSString *recordName; + +@property (nonatomic) NSData *encryptionKey; + +@property (nonatomic, nullable) NSString *relativeFilePath; + +@property (nonatomic, nullable) NSString *downloadFilePath; + +@property (nonatomic, nullable) NSNumber *uncompressedDataLength; + +@end + +#pragma mark - + +@implementation OWSBackupImportItem + +@end + +#pragma mark - + @interface OWSBackupImportJob () @property (nonatomic, nullable) OWSBackgroundTask *backgroundTask; -@property (nonatomic, nullable) OWSBackupStorage *backupStorage; +@property (nonatomic) OWSBackupEncryption *encryption; -// A map of "record name"-to-"file name". -@property (nonatomic) NSMutableDictionary *databaseRecordMap; - -// A map of "record name"-to-"file relative path". -@property (nonatomic) NSMutableDictionary *attachmentRecordMap; - -// A map of "record name"-to-"downloaded file path". -@property (nonatomic) NSMutableDictionary *downloadedFileMap; +@property (nonatomic) NSArray *databaseItems; +@property (nonatomic) NSArray *attachmentsItems; @end @@ -93,13 +108,13 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return; } - NSMutableArray *allRecordNames = [NSMutableArray new]; - [allRecordNames addObjectsFromArray:weakSelf.databaseRecordMap.allKeys]; - // TODO: We could skip attachments that have already been restored - // by previous "backup import" attempts. - [allRecordNames addObjectsFromArray:weakSelf.attachmentRecordMap.allKeys]; + OWSAssert(self.databaseItems); + OWSAssert(self.attachmentsItems); + NSMutableArray *allItems = [NSMutableArray new]; + [allItems addObjectsFromArray:self.databaseItems]; + [allItems addObjectsFromArray:self.attachmentsItems]; [weakSelf - downloadFilesFromCloud:allRecordNames + downloadFilesFromCloud:allItems completion:^(NSError *_Nullable fileDownloadError) { if (fileDownloadError) { [weakSelf failWithError:fileDownloadError]; @@ -157,6 +172,9 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe OWSProdLogAndFail(@"%@ Could not create jobTempDirPath.", self.logTag); return NO; } + + self.encryption = [[OWSBackupEncryption alloc] initWithJobTempDirPath:self.jobTempDirPath]; + return YES; } @@ -183,14 +201,15 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe } failure:^(NSError *error) { // The manifest file is critical so any error downloading it is unrecoverable. - OWSProdLogAndFail(@"%@ Could not download manifest.", self.logTag); + OWSProdLogAndFail(@"%@ Could not download manifest.", weakSelf.logTag); completion(error); }]; } -- (void)processManifest:(NSData *)manifestData completion:(OWSBackupJobBoolCompletion)completion +- (void)processManifest:(NSData *)manifestDataEncrypted completion:(OWSBackupJobBoolCompletion)completion { OWSAssert(completion); + OWSAssert(self.encryption); if (self.isComplete) { return; @@ -198,9 +217,16 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + NSData *_Nullable manifestDataDecrypted = + [self.encryption decryptDataAsData:manifestDataEncrypted encryptionKey:self.delegate.backupEncryptionKey]; + if (!manifestDataDecrypted) { + OWSProdLogAndFail(@"%@ Could not decrypt manifest.", self.logTag); + return completion(NO); + } + NSError *error; NSDictionary *_Nullable json = - [NSJSONSerialization JSONObjectWithData:manifestData options:0 error:&error]; + [NSJSONSerialization JSONObjectWithData:manifestDataDecrypted options:0 error:&error]; if (![json isKindOfClass:[NSDictionary class]]) { OWSProdLogAndFail(@"%@ Could not download manifest.", self.logTag); return completion(NO); @@ -208,49 +234,97 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe DDLogVerbose(@"%@ json: %@", self.logTag, json); - NSDictionary *_Nullable databaseRecordMap = json[kOWSBackup_ManifestKey_DatabaseFiles]; - NSDictionary *_Nullable attachmentRecordMap = json[kOWSBackup_ManifestKey_AttachmentFiles]; - NSString *_Nullable databaseKeySpecBase64 = json[kOWSBackup_ManifestKey_DatabaseKeySpec]; - if (!([databaseRecordMap isKindOfClass:[NSDictionary class]] && - [attachmentRecordMap isKindOfClass:[NSDictionary class]] && - [databaseKeySpecBase64 isKindOfClass:[NSString class]])) { - OWSProdLogAndFail(@"%@ Invalid manifest.", self.logTag); + NSArray *_Nullable databaseItems = + [self parseItems:json key:kOWSBackup_ManifestKey_DatabaseFiles]; + if (!databaseItems) { return completion(NO); } - NSData *_Nullable databaseKeySpec = [NSData dataFromBase64String:databaseKeySpecBase64]; - if (!databaseKeySpec) { - OWSProdLogAndFail(@"%@ Invalid manifest databaseKeySpec.", self.logTag); - return completion(NO); - } - if (![OWSBackupJob storeDatabaseKeySpec:databaseKeySpec keychainKey:kOWSBackup_ImportDatabaseKeySpec]) { - OWSProdLogAndFail(@"%@ Couldn't store databaseKeySpec from manifest.", self.logTag); + NSArray *_Nullable attachmentsItems = + [self parseItems:json key:kOWSBackup_ManifestKey_AttachmentFiles]; + if (!attachmentsItems) { return completion(NO); } - self.databaseRecordMap = [databaseRecordMap mutableCopy]; - self.attachmentRecordMap = [attachmentRecordMap mutableCopy]; + self.databaseItems = databaseItems; + self.attachmentsItems = attachmentsItems; return completion(YES); } -- (void)downloadFilesFromCloud:(NSMutableArray *)recordNames completion:(OWSBackupJobCompletion)completion +- (nullable NSArray *)parseItems:(id)json key:(NSString *)key { - OWSAssert(recordNames.count > 0); + OWSAssert(json); + OWSAssert(key.length); + + if (![json isKindOfClass:[NSArray class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key); + return nil; + } + NSArray *itemMaps = json[key]; + if (![itemMaps isKindOfClass:[NSArray class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key); + return nil; + } + NSMutableArray *items = [NSMutableArray new]; + for (NSDictionary *itemMap in itemMaps) { + if (![itemMap isKindOfClass:[NSDictionary class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid item: %@.", self.logTag, key); + return nil; + } + NSString *_Nullable recordName = itemMap[kOWSBackup_ManifestKey_RecordName]; + NSString *_Nullable encryptionKeyString = itemMap[kOWSBackup_ManifestKey_EncryptionKey]; + NSString *_Nullable relativeFilePath = itemMap[kOWSBackup_ManifestKey_RelativeFilePath]; + NSNumber *_Nullable uncompressedDataLength = itemMap[kOWSBackup_ManifestKey_DataSize]; + if (![recordName isKindOfClass:[NSString class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid recordName: %@.", self.logTag, key); + return nil; + } + if (![encryptionKeyString isKindOfClass:[NSString class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid encryptionKey: %@.", self.logTag, key); + return nil; + } + // relativeFilePath is an optional field. + if (relativeFilePath && ![relativeFilePath isKindOfClass:[NSString class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid relativeFilePath: %@.", self.logTag, key); + return nil; + } + NSData *_Nullable encryptionKey = [NSData dataFromBase64String:encryptionKeyString]; + if (!encryptionKey) { + OWSProdLogAndFail(@"%@ manifest has corrupt encryptionKey: %@.", self.logTag, key); + return nil; + } + // uncompressedDataLength is an optional field. + if (uncompressedDataLength && ![uncompressedDataLength isKindOfClass:[NSNumber class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid uncompressedDataLength: %@.", self.logTag, key); + return nil; + } + + OWSBackupImportItem *item = [OWSBackupImportItem new]; + item.recordName = recordName; + item.encryptionKey = encryptionKey; + item.relativeFilePath = relativeFilePath; + item.uncompressedDataLength = uncompressedDataLength; + [items addObject:item]; + } + return items; +} + +- (void)downloadFilesFromCloud:(NSMutableArray *)items + completion:(OWSBackupJobCompletion)completion +{ + OWSAssert(items.count > 0); OWSAssert(completion); DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - // A map of "record name"-to-"downloaded file path". - self.downloadedFileMap = [NSMutableDictionary new]; - - [self downloadNextFileFromCloud:recordNames recordCount:recordNames.count completion:completion]; + [self downloadNextItemFromCloud:items recordCount:items.count completion:completion]; } -- (void)downloadNextFileFromCloud:(NSMutableArray *)recordNames +- (void)downloadNextItemFromCloud:(NSMutableArray *)items recordCount:(NSUInteger)recordCount completion:(OWSBackupJobCompletion)completion { - OWSAssert(recordNames); + OWSAssert(items); OWSAssert(completion); if (self.isComplete) { @@ -258,43 +332,43 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return completion(nil); } - if (recordNames.count < 1) { + if (items.count < 1) { // All downloads are complete; exit. return completion(nil); } - NSString *recordName = recordNames.lastObject; - [recordNames removeLastObject]; + OWSBackupImportItem *item = items.lastObject; + [items removeLastObject]; + CGFloat progress = (recordCount > 0 ? ((recordCount - items.count) / (CGFloat)recordCount) : 0.f); [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_DOWNLOAD", @"Indicates that the backup import data is being downloaded.") - progress:@((recordCount - recordNames.count) / (CGFloat)recordCount)]; - - if (![recordName isKindOfClass:[NSString class]]) { - DDLogError(@"%@ invalid record name in manifest: %@", self.logTag, [recordName class]); - // Invalid record name in the manifest. This may be recoverable. - // Ignore this for now and proceed with the other downloads. - return [self downloadNextFileFromCloud:recordNames recordCount:recordCount completion:completion]; - } + progress:@(progress)]; // Use a predictable file path so that multiple "import backup" attempts // will leverage successful file downloads from previous attempts. - NSString *tempFilePath = [self.jobTempDirPath stringByAppendingPathComponent:recordName]; + // + // TODO: This will also require imports using a predictable jobTempDirPath. + NSString *tempFilePath = [self.jobTempDirPath stringByAppendingPathComponent:item.recordName]; // Skip redundant file download. if ([NSFileManager.defaultManager fileExistsAtPath:tempFilePath]) { [OWSFileSystem protectFileOrFolderAtPath:tempFilePath]; - self.downloadedFileMap[recordName] = tempFilePath; - [self downloadNextFileFromCloud:recordNames recordCount:recordCount completion:completion]; + + item.downloadFilePath = tempFilePath; + + [self downloadNextItemFromCloud:items recordCount:recordCount completion:completion]; return; } - [OWSBackupAPI downloadFileFromCloudWithRecordName:recordName + __weak OWSBackupImportJob *weakSelf = self; + [OWSBackupAPI downloadFileFromCloudWithRecordName:item.recordName toFileUrl:[NSURL fileURLWithPath:tempFilePath] success:^{ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ [OWSFileSystem protectFileOrFolderAtPath:tempFilePath]; - self.downloadedFileMap[recordName] = tempFilePath; - [self downloadNextFileFromCloud:recordNames recordCount:recordCount completion:completion]; + item.downloadFilePath = tempFilePath; + + [weakSelf downloadNextItemFromCloud:items recordCount:recordCount completion:completion]; }); } failure:^(NSError *error) { @@ -304,28 +378,45 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe - (void)restoreAttachmentFiles { - DDLogVerbose(@"%@ %s: %zd", self.logTag, __PRETTY_FUNCTION__, self.attachmentRecordMap.count); + DDLogVerbose(@"%@ %s: %zd", self.logTag, __PRETTY_FUNCTION__, self.attachmentsItems.count); NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder]; NSUInteger count = 0; - for (NSString *recordName in self.attachmentRecordMap) { + for (OWSBackupImportItem *item in self.attachmentsItems) { if (self.isComplete) { return; } + if (item.recordName.length < 1) { + DDLogError(@"%@ attachment was not downloaded.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + continue; + } + if (item.relativeFilePath.length < 1) { + DDLogError(@"%@ attachment missing relative file path.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + continue; + } count++; [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_FILES", @"Indicates that the backup import data is being restored.") - progress:@(count / (CGFloat)self.attachmentRecordMap.count)]; + progress:@(count / (CGFloat)self.attachmentsItems.count)]; - - NSString *dstRelativePath = self.attachmentRecordMap[recordName]; - if (! - [self restoreFileWithRecordName:recordName dstRelativePath:dstRelativePath dstDirPath:attachmentsDirPath]) { + NSString *dstFilePath = [attachmentsDirPath stringByAppendingPathComponent:item.relativeFilePath]; + if ([NSFileManager.defaultManager fileExistsAtPath:dstFilePath]) { + DDLogError(@"%@ skipping redundant file restore: %@.", self.logTag, dstFilePath); + continue; + } + if (![self.encryption decryptFileAsFile:item.downloadFilePath + dstFilePath:dstFilePath + encryptionKey:item.encryptionKey]) { + DDLogError(@"%@ attachment could not be restored.", self.logTag); // Attachment-related errors are recoverable and can be ignored. continue; } + + DDLogError(@"%@ restored file: %@.", self.logTag, item.relativeFilePath); } } @@ -335,84 +426,16 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_DATABASE", - @"Indicates that the backup database is being restored.") - progress:nil]; - - NSString *jobDatabaseDirPath = [self.jobTempDirPath stringByAppendingPathComponent:@"database"]; - if (![OWSFileSystem ensureDirectoryExists:jobDatabaseDirPath]) { - OWSProdLogAndFail(@"%@ Could not create jobDatabaseDirPath.", self.logTag); - return completion(NO); - } - - for (NSString *recordName in self.databaseRecordMap) { - if (self.isComplete) { - return completion(NO); - } - - NSString *dstRelativePath = self.databaseRecordMap[recordName]; - if (! - [self restoreFileWithRecordName:recordName dstRelativePath:dstRelativePath dstDirPath:jobDatabaseDirPath]) { - // Database-related errors are unrecoverable. - return completion(NO); - } - } - - BackupStorageKeySpecBlock keySpecBlock = ^{ - NSData *_Nullable databaseKeySpec = - [OWSBackupJob loadDatabaseKeySpecWithKeychainKey:kOWSBackup_ImportDatabaseKeySpec]; - if (!databaseKeySpec) { - OWSProdLogAndFail(@"%@ Could not load database keyspec for import.", self.logTag); - } - return databaseKeySpec; - }; - self.backupStorage = - [[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock]; - if (!self.backupStorage) { - OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag); - return completion(NO); - } - - // TODO: Do we really need to run these registrations on the main thread? - __weak OWSBackupImportJob *weakSelf = self; - dispatch_async(dispatch_get_main_queue(), ^{ - [weakSelf.backupStorage runSyncRegistrations]; - [weakSelf.backupStorage runAsyncRegistrationsWithCompletion:^{ - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - [weakSelf restoreDatabaseContents:completion]; - }); - }]; - }); -} - -- (void)restoreDatabaseContents:(OWSBackupJobBoolCompletion)completion -{ - OWSAssert(self.backupStorage); - OWSAssert(completion); - - DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - if (self.isComplete) { return completion(NO); } - YapDatabaseConnection *_Nullable tempDBConnection = self.backupStorage.newDatabaseConnection; - if (!tempDBConnection) { - OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag); - return completion(NO); - } - YapDatabaseConnection *_Nullable primaryDBConnection = self.primaryStorage.newDatabaseConnection; - if (!primaryDBConnection) { - OWSProdLogAndFail(@"%@ Could not create primaryDBConnection.", self.logTag); + YapDatabaseConnection *_Nullable dbConnection = self.primaryStorage.newDatabaseConnection; + if (!dbConnection) { + OWSProdLogAndFail(@"%@ Could not create dbConnection.", self.logTag); return completion(NO); } - NSDictionary *collectionTypeMap = @{ - [TSThread collection] : [TSThread class], - [TSAttachment collection] : [TSAttachment class], - [TSInteraction collection] : [TSInteraction class], - [OWSDatabaseMigration collection] : [OWSDatabaseMigration class], - }; // Order matters here. NSArray *collectionsToRestore = @[ [TSThread collection], @@ -425,85 +448,123 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe NSMutableDictionary *restoredEntityCounts = [NSMutableDictionary new]; __block unsigned long long copiedEntities = 0; __block BOOL aborted = NO; - [tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *srcTransaction) { - [primaryDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { - if (![srcTransaction boolForKey:kOWSBackup_Snapshot_ValidKey - inCollection:kOWSBackup_Snapshot_Collection - defaultValue:NO]) { - DDLogError(@"%@ invalid database.", self.logTag); + [dbConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { + for (NSString *collection in collectionsToRestore) { + if ([collection isEqualToString:[OWSDatabaseMigration collection]]) { + // It's okay if there are existing migrations; we'll clear those + // before restoring. + continue; + } + if ([transaction numberOfKeysInCollection:collection] > 0) { + DDLogError(@"%@ unexpected contents in database (%@).", self.logTag, collection); + } + } + + // Clear existing database contents. + // + // This should be safe since we only ever import into an empty database. + // + // Note that if the app receives a message after registering and before restoring + // backup, it will be lost. + // + // Note that this will clear all migrations. + for (NSString *collection in collectionsToRestore) { + [transaction removeAllObjectsInCollection:collection]; + } + + NSUInteger count = 0; + for (OWSBackupImportItem *item in self.databaseItems) { + if (self.isComplete) { + return; + } + if (item.recordName.length < 1) { + DDLogError(@"%@ database snapshot was not downloaded.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + if (!item.uncompressedDataLength || item.uncompressedDataLength.unsignedIntValue < 1) { + DDLogError(@"%@ database snapshot missing size.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + // Database-related errors are unrecoverable. aborted = YES; return completion(NO); } - for (NSString *collection in collectionsToRestore) { - if ([collection isEqualToString:[OWSDatabaseMigration collection]]) { - // It's okay if there are existing migrations; we'll clear those - // before restoring. - continue; + count++; + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_DATABASE", + @"Indicates that the backup database is being restored.") + progress:@(count / (CGFloat)self.databaseItems.count)]; + + NSData *_Nullable compressedData = + [self.encryption decryptFileAsData:item.downloadFilePath encryptionKey:item.encryptionKey]; + if (!compressedData) { + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + NSData *_Nullable uncompressedData = + [self.encryption decompressData:compressedData + uncompressedDataLength:item.uncompressedDataLength.unsignedIntValue]; + if (!uncompressedData) { + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + OWSSignalServiceProtosBackupSnapshot *_Nullable entities = + [OWSSignalServiceProtosBackupSnapshot parseFromData:uncompressedData]; + if (!entities || entities.entity.count < 1) { + DDLogError(@"%@ missing entities.", self.logTag); + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + for (OWSSignalServiceProtosBackupSnapshotBackupEntity *entity in entities.entity) { + NSData *_Nullable entityData = entity.entityData; + if (entityData.length < 1) { + DDLogError(@"%@ missing entity data.", self.logTag); + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); } - if ([dstTransaction numberOfKeysInCollection:collection] > 0) { - DDLogError(@"%@ unexpected contents in database (%@).", self.logTag, collection); + + __block TSYapDatabaseObject *object = nil; + @try { + NSKeyedUnarchiver *unarchiver = [[NSKeyedUnarchiver alloc] initForReadingWithData:entityData]; + object = [unarchiver decodeObjectForKey:@"root"]; + if (![object isKindOfClass:[object class]]) { + DDLogError(@"%@ invalid decoded entity: %@.", self.logTag, [object class]); + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + } @catch (NSException *exception) { + DDLogError(@"%@ could not decode entity.", self.logTag); + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); } - } - // Clear existing database contents. - // - // This should be safe since we only ever import into an empty database. - // - // Note that if the app receives a message after registering and before restoring - // backup, it will be lost. - // - // Note that this will clear all migrations. - for (NSString *collection in collectionsToRestore) { - [dstTransaction removeAllObjectsInCollection:collection]; + [object saveWithTransaction:transaction]; + copiedEntities++; + NSString *collection = [object.class collection]; + NSUInteger restoredEntityCount = restoredEntityCounts[collection].unsignedIntValue; + restoredEntityCounts[collection] = @(restoredEntityCount + 1); } - - // Copy database entities. - for (NSString *collection in collectionsToRestore) { - [srcTransaction enumerateKeysAndObjectsInCollection:collection - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - aborted = YES; - return; - } - Class expectedType = collectionTypeMap[collection]; - OWSAssert(expectedType); - if (![object isKindOfClass:expectedType]) { - OWSProdLogAndFail(@"%@ unexpected class: %@ != %@", - self.logTag, - [object class], - expectedType); - return; - } - TSYapDatabaseObject *databaseObject = object; - [databaseObject saveWithTransaction:dstTransaction]; - - NSUInteger count - = restoredEntityCounts[collection].unsignedIntValue; - restoredEntityCounts[collection] = @(count + 1); - copiedEntities++; - }]; - } - }]; + } }]; - if (aborted) { + if (self.isComplete || aborted) { return; } - for (NSString *collection in collectionsToRestore) { - Class expectedType = collectionTypeMap[collection]; - OWSAssert(expectedType); - DDLogInfo(@"%@ copied %@ (%@): %@", self.logTag, expectedType, collection, restoredEntityCounts[collection]); + for (NSString *collection in restoredEntityCounts) { + DDLogInfo(@"%@ copied %@: %@", self.logTag, collection, restoredEntityCounts[collection]); } DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities); - [self.backupStorage logFileSizes]; - - // Close the database. - tempDBConnection = nil; - self.backupStorage = nil; + [self.primaryStorage logFileSizes]; completion(YES); } @@ -530,45 +591,6 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe }); } -- (BOOL)restoreFileWithRecordName:(NSString *)recordName - dstRelativePath:(NSString *)dstRelativePath - dstDirPath:(NSString *)dstDirPath -{ - OWSAssert(recordName); - OWSAssert(dstDirPath.length > 0); - - DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - - if (![recordName isKindOfClass:[NSString class]]) { - DDLogError(@"%@ invalid record name in manifest: %@", self.logTag, [recordName class]); - return NO; - } - if (![dstRelativePath isKindOfClass:[NSString class]]) { - DDLogError(@"%@ invalid dstRelativePath in manifest: %@", self.logTag, [recordName class]); - return NO; - } - NSString *dstFilePath = [dstDirPath stringByAppendingPathComponent:dstRelativePath]; - if ([NSFileManager.defaultManager fileExistsAtPath:dstFilePath]) { - DDLogError(@"%@ skipping redundant file restore: %@.", self.logTag, dstFilePath); - return YES; - } - NSString *downloadedFilePath = self.downloadedFileMap[recordName]; - if (![NSFileManager.defaultManager fileExistsAtPath:downloadedFilePath]) { - DDLogError(@"%@ missing downloaded attachment file.", self.logTag); - return NO; - } - NSError *error; - BOOL success = [NSFileManager.defaultManager moveItemAtPath:downloadedFilePath toPath:dstFilePath error:&error]; - if (!success || error) { - DDLogError(@"%@ could not restore attachment file.", self.logTag); - return NO; - } - - DDLogError(@"%@ restored file: %@ (%@).", self.logTag, dstFilePath, dstRelativePath); - - return YES; -} - @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackupJob.h b/Signal/src/util/OWSBackupJob.h index 146e8ba7b9..4476cdcfe9 100644 --- a/Signal/src/util/OWSBackupJob.h +++ b/Signal/src/util/OWSBackupJob.h @@ -9,6 +9,7 @@ extern NSString *const kOWSBackup_ManifestKey_AttachmentFiles; extern NSString *const kOWSBackup_ManifestKey_RecordName; extern NSString *const kOWSBackup_ManifestKey_EncryptionKey; extern NSString *const kOWSBackup_ManifestKey_RelativeFilePath; +extern NSString *const kOWSBackup_ManifestKey_DataSize; typedef void (^OWSBackupJobBoolCompletion)(BOOL success); typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error); @@ -62,12 +63,6 @@ typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error); - (void)failWithError:(NSError *)error; - (void)updateProgressWithDescription:(nullable NSString *)description progress:(nullable NSNumber *)progress; -#pragma mark - Database KeySpec - -+ (nullable NSData *)loadDatabaseKeySpecWithKeychainKey:(NSString *)keychainKey; -+ (BOOL)storeDatabaseKeySpec:(NSData *)data keychainKey:(NSString *)keychainKey; -+ (BOOL)generateRandomDatabaseKeySpecWithKeychainKey:(NSString *)keychainKey; - @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackupJob.m b/Signal/src/util/OWSBackupJob.m index d298979abd..a0b9c933ce 100644 --- a/Signal/src/util/OWSBackupJob.m +++ b/Signal/src/util/OWSBackupJob.m @@ -16,6 +16,7 @@ NSString *const kOWSBackup_ManifestKey_AttachmentFiles = @"attachment_files"; NSString *const kOWSBackup_ManifestKey_RecordName = @"record_name"; NSString *const kOWSBackup_ManifestKey_EncryptionKey = @"encryption_key"; NSString *const kOWSBackup_ManifestKey_RelativeFilePath = @"relative_file_path"; +NSString *const kOWSBackup_ManifestKey_DataSize = @"data_size"; NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; @@ -146,51 +147,6 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; }); } -#pragma mark - Database KeySpec - -+ (nullable NSData *)loadDatabaseKeySpecWithKeychainKey:(NSString *)keychainKey -{ - OWSAssert(keychainKey.length > 0); - - NSError *error; - NSData *_Nullable value = - [SAMKeychain passwordDataForService:kOWSBackup_KeychainService account:keychainKey error:&error]; - if (!value || error) { - DDLogError(@"%@ could not load database keyspec: %@", self.logTag, error); - } - return value; -} - -+ (BOOL)storeDatabaseKeySpec:(NSData *)data keychainKey:(NSString *)keychainKey -{ - OWSAssert(keychainKey.length > 0); - OWSAssert(data.length > 0); - - NSError *error; - [SAMKeychain setAccessibilityType:kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly]; - BOOL success = - [SAMKeychain setPasswordData:data forService:kOWSBackup_KeychainService account:keychainKey error:&error]; - if (!success || error) { - OWSFail(@"%@ Could not store database keyspec: %@.", self.logTag, error); - return NO; - } else { - return YES; - } -} - -+ (BOOL)generateRandomDatabaseKeySpecWithKeychainKey:(NSString *)keychainKey -{ - OWSAssert(keychainKey.length > 0); - - NSData *_Nullable databaseKeySpec = [Randomness generateRandomBytes:(int)kSQLCipherKeySpecLength]; - if (!databaseKeySpec) { - OWSFail(@"%@ Could not generate database keyspec.", self.logTag); - return NO; - } - - return [self storeDatabaseKeySpec:databaseKeySpec keychainKey:keychainKey]; -} - @end NS_ASSUME_NONNULL_END diff --git a/SignalServiceKit/protobuf/Makefile b/SignalServiceKit/protobuf/Makefile index d7b4c47b7c..42772b7db3 100644 --- a/SignalServiceKit/protobuf/Makefile +++ b/SignalServiceKit/protobuf/Makefile @@ -1,9 +1,9 @@ # See README.md in this dir for prerequisite setup. - + PROTOC=protoc \ --plugin=/usr/local/bin/protoc-gen-objc \ - --proto_path="${HOME}/src/WhisperSystems/protobuf-objc/src/compiler/" \ - --proto_path="${HOME}/src/WhisperSystems/protobuf-objc/src/compiler/google/protobuf/" \ + --proto_path="${HOME}/code/workspace/ows/protobuf-objc/src/compiler/" \ + --proto_path="${HOME}/code/workspace/ows/protobuf-objc/src/compiler/google/protobuf/" \ --proto_path='./' all: signal_service_proto provisioning_protos fingerprint_protos diff --git a/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto b/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto index e4ab31e8d9..c2e01fe34e 100644 --- a/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto +++ b/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto @@ -251,7 +251,7 @@ message BackupSnapshot ATTACHMENT = 4; } optional Type type = 1; - optional bytes data = 2; + optional bytes entityData = 2; } repeated BackupEntity entity = 1; diff --git a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h index 59e6837d19..494e171aec 100644 --- a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h +++ b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h @@ -2266,18 +2266,18 @@ NSString *NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(OWSSi @end #define BackupEntity_type @"type" -#define BackupEntity_data @"data" +#define BackupEntity_entityData @"entityData" @interface OWSSignalServiceProtosBackupSnapshotBackupEntity : PBGeneratedMessage { @private - BOOL hasData_:1; + BOOL hasEntityData_:1; BOOL hasType_:1; - NSData* data; + NSData* entityData; OWSSignalServiceProtosBackupSnapshotBackupEntityType type; } - (BOOL) hasType; -- (BOOL) hasData; +- (BOOL) hasEntityData; @property (readonly) OWSSignalServiceProtosBackupSnapshotBackupEntityType type; -@property (readonly, strong) NSData* data; +@property (readonly, strong) NSData* entityData; + (instancetype) defaultInstance; - (instancetype) defaultInstance; @@ -2319,10 +2319,10 @@ NSString *NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(OWSSi - (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setType:(OWSSignalServiceProtosBackupSnapshotBackupEntityType) value; - (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearType; -- (BOOL) hasData; -- (NSData*) data; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setData:(NSData*) value; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearData; +- (BOOL) hasEntityData; +- (NSData*) entityData; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setEntityData:(NSData*) value; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearEntityData; @end @interface OWSSignalServiceProtosBackupSnapshotBuilder : PBGeneratedMessageBuilder { diff --git a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m index 17911e653b..1770d2abe6 100644 --- a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m +++ b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m @@ -9830,7 +9830,7 @@ static OWSSignalServiceProtosBackupSnapshot* defaultOWSSignalServiceProtosBackup @interface OWSSignalServiceProtosBackupSnapshotBackupEntity () @property OWSSignalServiceProtosBackupSnapshotBackupEntityType type; -@property (strong) NSData* data; +@property (strong) NSData* entityData; @end @implementation OWSSignalServiceProtosBackupSnapshotBackupEntity @@ -9842,17 +9842,17 @@ static OWSSignalServiceProtosBackupSnapshot* defaultOWSSignalServiceProtosBackup hasType_ = !!_value_; } @synthesize type; -- (BOOL) hasData { - return !!hasData_; +- (BOOL) hasEntityData { + return !!hasEntityData_; } -- (void) setHasData:(BOOL) _value_ { - hasData_ = !!_value_; +- (void) setHasEntityData:(BOOL) _value_ { + hasEntityData_ = !!_value_; } -@synthesize data; +@synthesize entityData; - (instancetype) init { if ((self = [super init])) { self.type = OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown; - self.data = [NSData data]; + self.entityData = [NSData data]; } return self; } @@ -9875,8 +9875,8 @@ static OWSSignalServiceProtosBackupSnapshotBackupEntity* defaultOWSSignalService if (self.hasType) { [output writeEnum:1 value:self.type]; } - if (self.hasData) { - [output writeData:2 value:self.data]; + if (self.hasEntityData) { + [output writeData:2 value:self.entityData]; } [self.unknownFields writeToCodedOutputStream:output]; } @@ -9890,8 +9890,8 @@ static OWSSignalServiceProtosBackupSnapshotBackupEntity* defaultOWSSignalService if (self.hasType) { size_ += computeEnumSize(1, self.type); } - if (self.hasData) { - size_ += computeDataSize(2, self.data); + if (self.hasEntityData) { + size_ += computeDataSize(2, self.entityData); } size_ += self.unknownFields.serializedSize; memoizedSerializedSize = size_; @@ -9931,8 +9931,8 @@ static OWSSignalServiceProtosBackupSnapshotBackupEntity* defaultOWSSignalService if (self.hasType) { [output appendFormat:@"%@%@: %@\n", indent, @"type", NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(self.type)]; } - if (self.hasData) { - [output appendFormat:@"%@%@: %@\n", indent, @"data", self.data]; + if (self.hasEntityData) { + [output appendFormat:@"%@%@: %@\n", indent, @"entityData", self.entityData]; } [self.unknownFields writeDescriptionTo:output withIndent:indent]; } @@ -9940,8 +9940,8 @@ static OWSSignalServiceProtosBackupSnapshotBackupEntity* defaultOWSSignalService if (self.hasType) { [dictionary setObject: @(self.type) forKey: @"type"]; } - if (self.hasData) { - [dictionary setObject: self.data forKey: @"data"]; + if (self.hasEntityData) { + [dictionary setObject: self.entityData forKey: @"entityData"]; } [self.unknownFields storeInDictionary:dictionary]; } @@ -9956,8 +9956,8 @@ static OWSSignalServiceProtosBackupSnapshotBackupEntity* defaultOWSSignalService return self.hasType == otherMessage.hasType && (!self.hasType || self.type == otherMessage.type) && - self.hasData == otherMessage.hasData && - (!self.hasData || [self.data isEqual:otherMessage.data]) && + self.hasEntityData == otherMessage.hasEntityData && + (!self.hasEntityData || [self.entityData isEqual:otherMessage.entityData]) && (self.unknownFields == otherMessage.unknownFields || (self.unknownFields != nil && [self.unknownFields isEqual:otherMessage.unknownFields])); } - (NSUInteger) hash { @@ -9965,8 +9965,8 @@ static OWSSignalServiceProtosBackupSnapshotBackupEntity* defaultOWSSignalService if (self.hasType) { hashCode = hashCode * 31 + self.type; } - if (self.hasData) { - hashCode = hashCode * 31 + [self.data hash]; + if (self.hasEntityData) { + hashCode = hashCode * 31 + [self.entityData hash]; } hashCode = hashCode * 31 + [self.unknownFields hash]; return hashCode; @@ -10043,8 +10043,8 @@ NSString *NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(OWSSi if (other.hasType) { [self setType:other.type]; } - if (other.hasData) { - [self setData:other.data]; + if (other.hasEntityData) { + [self setEntityData:other.entityData]; } [self mergeUnknownFields:other.unknownFields]; return self; @@ -10077,7 +10077,7 @@ NSString *NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(OWSSi break; } case 18: { - [self setData:[input readData]]; + [self setEntityData:[input readData]]; break; } } @@ -10099,20 +10099,20 @@ NSString *NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(OWSSi resultBackupEntity.type = OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown; return self; } -- (BOOL) hasData { - return resultBackupEntity.hasData; +- (BOOL) hasEntityData { + return resultBackupEntity.hasEntityData; } -- (NSData*) data { - return resultBackupEntity.data; +- (NSData*) entityData { + return resultBackupEntity.entityData; } -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setData:(NSData*) value { - resultBackupEntity.hasData = YES; - resultBackupEntity.data = value; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setEntityData:(NSData*) value { + resultBackupEntity.hasEntityData = YES; + resultBackupEntity.entityData = value; return self; } -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearData { - resultBackupEntity.hasData = NO; - resultBackupEntity.data = [NSData data]; +- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearEntityData { + resultBackupEntity.hasEntityData = NO; + resultBackupEntity.entityData = [NSData data]; return self; } @end diff --git a/SignalServiceKit/src/Storage/OWSStorage.m b/SignalServiceKit/src/Storage/OWSStorage.m index 641749cfbc..73fc3680db 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.m +++ b/SignalServiceKit/src/Storage/OWSStorage.m @@ -46,14 +46,6 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); #pragma mark - -//@interface OWSDatabaseConnection : YapDatabaseConnection -// -//@property (atomic, weak) id delegate; -// -//@end - -#pragma mark - - @implementation OWSDatabaseConnection - (id)initWithDatabase:(YapDatabase *)database delegate:(id)delegate From fed524ba16367441fe8b32307aa6fc91a669800b Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Fri, 16 Mar 2018 14:04:01 -0300 Subject: [PATCH 04/13] Rework database snapshot representation, encryption, etc. --- Signal.xcodeproj/project.pbxproj | 12 ++--- Signal/src/util/OWSBackupExportJob.m | 46 +++++++++---------- .../{OWSBackupEncryption.h => OWSBackupIO.h} | 2 +- .../{OWSBackupEncryption.m => OWSBackupIO.m} | 6 +-- Signal/src/util/OWSBackupImportJob.m | 22 ++++----- Signal/src/util/OWSBackupJob.m | 1 - 6 files changed, 44 insertions(+), 45 deletions(-) rename Signal/src/util/{OWSBackupEncryption.h => OWSBackupIO.h} (97%) rename Signal/src/util/{OWSBackupEncryption.m => OWSBackupIO.m} (98%) diff --git a/Signal.xcodeproj/project.pbxproj b/Signal.xcodeproj/project.pbxproj index 2ebc0623e9..18f0850457 100644 --- a/Signal.xcodeproj/project.pbxproj +++ b/Signal.xcodeproj/project.pbxproj @@ -40,7 +40,7 @@ 340FC8C7204DE64D007AEB0F /* OWSBackupAPI.swift in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8C6204DE64D007AEB0F /* OWSBackupAPI.swift */; }; 340FC8CA20517B84007AEB0F /* OWSBackupImportJob.m in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8C820517B84007AEB0F /* OWSBackupImportJob.m */; }; 340FC8CD20518C77007AEB0F /* OWSBackupJob.m in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8CC20518C76007AEB0F /* OWSBackupJob.m */; }; - 340FC8D0205BF2FA007AEB0F /* OWSBackupEncryption.m in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8CE205BF2FA007AEB0F /* OWSBackupEncryption.m */; }; + 340FC8D0205BF2FA007AEB0F /* OWSBackupIO.m in Sources */ = {isa = PBXBuildFile; fileRef = 340FC8CE205BF2FA007AEB0F /* OWSBackupIO.m */; }; 341F2C0F1F2B8AE700D07D6B /* DebugUIMisc.m in Sources */ = {isa = PBXBuildFile; fileRef = 341F2C0E1F2B8AE700D07D6B /* DebugUIMisc.m */; }; 3430FE181F7751D4000EC51B /* GiphyAPI.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3430FE171F7751D4000EC51B /* GiphyAPI.swift */; }; 34330A5A1E7875FB00DF2FB9 /* fontawesome-webfont.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 34330A591E7875FB00DF2FB9 /* fontawesome-webfont.ttf */; }; @@ -589,8 +589,8 @@ 340FC8C920517B84007AEB0F /* OWSBackupImportJob.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSBackupImportJob.h; sourceTree = ""; }; 340FC8CB20518C76007AEB0F /* OWSBackupJob.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSBackupJob.h; sourceTree = ""; }; 340FC8CC20518C76007AEB0F /* OWSBackupJob.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSBackupJob.m; sourceTree = ""; }; - 340FC8CE205BF2FA007AEB0F /* OWSBackupEncryption.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSBackupEncryption.m; sourceTree = ""; }; - 340FC8CF205BF2FA007AEB0F /* OWSBackupEncryption.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSBackupEncryption.h; sourceTree = ""; }; + 340FC8CE205BF2FA007AEB0F /* OWSBackupIO.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSBackupIO.m; sourceTree = ""; }; + 340FC8CF205BF2FA007AEB0F /* OWSBackupIO.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSBackupIO.h; sourceTree = ""; }; 341458471FBE11C4005ABCF9 /* fa */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = fa; path = translations/fa.lproj/Localizable.strings; sourceTree = ""; }; 341F2C0D1F2B8AE700D07D6B /* DebugUIMisc.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DebugUIMisc.h; sourceTree = ""; }; 341F2C0E1F2B8AE700D07D6B /* DebugUIMisc.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = DebugUIMisc.m; sourceTree = ""; }; @@ -1958,8 +1958,8 @@ 34A9105E1FFEB113000C4745 /* OWSBackup.h */, 34A9105F1FFEB114000C4745 /* OWSBackup.m */, 340FC8C6204DE64D007AEB0F /* OWSBackupAPI.swift */, - 340FC8CF205BF2FA007AEB0F /* OWSBackupEncryption.h */, - 340FC8CE205BF2FA007AEB0F /* OWSBackupEncryption.m */, + 340FC8CF205BF2FA007AEB0F /* OWSBackupIO.h */, + 340FC8CE205BF2FA007AEB0F /* OWSBackupIO.m */, 340FC8BE204DB7D1007AEB0F /* OWSBackupExportJob.h */, 340FC8BF204DB7D2007AEB0F /* OWSBackupExportJob.m */, 340FC8C920517B84007AEB0F /* OWSBackupImportJob.h */, @@ -3177,7 +3177,7 @@ 34D1F0B71F87F8850066283D /* OWSGenericAttachmentView.m in Sources */, 34B3F8801E8DF1700035BE1A /* InviteFlow.swift in Sources */, 457C87B82032645C008D52D6 /* DebugUINotifications.swift in Sources */, - 340FC8D0205BF2FA007AEB0F /* OWSBackupEncryption.m in Sources */, + 340FC8D0205BF2FA007AEB0F /* OWSBackupIO.m in Sources */, 458E38371D668EBF0094BD24 /* OWSDeviceProvisioningURLParser.m in Sources */, 4517642B1DE939FD00EDB8B9 /* ContactCell.swift in Sources */, 340FC8AB204DAC8D007AEB0F /* DomainFrontingCountryViewController.m in Sources */, diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index 49c364df3b..aba34796a0 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -3,7 +3,7 @@ // #import "OWSBackupExportJob.h" -#import "OWSBackupEncryption.h" +#import "OWSBackupIO.h" #import "OWSDatabaseMigration.h" #import "OWSSignalServiceProtos.pb.h" #import "Signal-Swift.h" @@ -65,7 +65,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OWSDBExportStream : NSObject -@property (nonatomic) OWSBackupEncryption *encryption; +@property (nonatomic) OWSBackupIO *backupIO; @property (nonatomic) NSMutableArray *exportItems; @@ -83,16 +83,16 @@ NS_ASSUME_NONNULL_BEGIN @implementation OWSDBExportStream -- (instancetype)initWithEncryption:(OWSBackupEncryption *)encryption +- (instancetype)initWithBackupIO:(OWSBackupIO *)backupIO { if (!(self = [super init])) { return self; } - OWSAssert(encryption); + OWSAssert(backupIO); self.exportItems = [NSMutableArray new]; - self.encryption = encryption; + self.backupIO = backupIO; return self; } @@ -146,9 +146,9 @@ NS_ASSUME_NONNULL_BEGIN return NO; } - NSData *compressedData = [self.encryption compressData:uncompressedData]; + NSData *compressedData = [self.backupIO compressData:uncompressedData]; - OWSBackupEncryptedItem *_Nullable encryptedItem = [self.encryption encryptDataAsTempFile:compressedData]; + OWSBackupEncryptedItem *_Nullable encryptedItem = [self.backupIO encryptDataAsTempFile:compressedData]; if (!encryptedItem) { OWSProdLogAndFail(@"%@ couldn't encrypt database snapshot.", self.logTag); return NO; @@ -167,7 +167,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OWSAttachmentExport : NSObject -@property (nonatomic) OWSBackupEncryption *encryption; +@property (nonatomic) OWSBackupIO *backupIO; @property (nonatomic) NSString *attachmentId; @property (nonatomic) NSString *attachmentFilePath; @property (nonatomic, nullable) NSString *relativeFilePath; @@ -181,19 +181,19 @@ NS_ASSUME_NONNULL_BEGIN @implementation OWSAttachmentExport -- (instancetype)initWithEncryption:(OWSBackupEncryption *)encryption - attachmentId:(NSString *)attachmentId - attachmentFilePath:(NSString *)attachmentFilePath +- (instancetype)initWithBackupIO:(OWSBackupIO *)backupIO + attachmentId:(NSString *)attachmentId + attachmentFilePath:(NSString *)attachmentFilePath { if (!(self = [super init])) { return self; } - OWSAssert(encryption); + OWSAssert(backupIO); OWSAssert(attachmentId.length > 0); OWSAssert(attachmentFilePath.length > 0); - self.encryption = encryption; + self.backupIO = backupIO; self.attachmentId = attachmentId; self.attachmentFilePath = attachmentFilePath; @@ -225,7 +225,7 @@ NS_ASSUME_NONNULL_BEGIN } self.relativeFilePath = relativeFilePath; - OWSBackupEncryptedItem *_Nullable encryptedItem = [self.encryption encryptFileAsTempFile:self.attachmentFilePath]; + OWSBackupEncryptedItem *_Nullable encryptedItem = [self.backupIO encryptFileAsTempFile:self.attachmentFilePath]; if (!encryptedItem) { DDLogError(@"%@ attachment could not be encrypted.", self.logTag); OWSFail(@"%@ attachment could not be encrypted: %@", self.logTag, self.attachmentFilePath); @@ -243,7 +243,7 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic, nullable) OWSBackgroundTask *backgroundTask; -@property (nonatomic) OWSBackupEncryption *encryption; +@property (nonatomic) OWSBackupIO *backupIO; @property (nonatomic) NSMutableArray *unsavedDatabaseItems; @@ -338,7 +338,7 @@ NS_ASSUME_NONNULL_BEGIN return completion(NO); } - self.encryption = [[OWSBackupEncryption alloc] initWithJobTempDirPath:self.jobTempDirPath]; + self.backupIO = [[OWSBackupIO alloc] initWithJobTempDirPath:self.jobTempDirPath]; // We need to verify that we have a valid account. // Otherwise, if we re-register on another device, we @@ -365,7 +365,7 @@ NS_ASSUME_NONNULL_BEGIN - (BOOL)exportDatabase { - OWSAssert(self.encryption); + OWSAssert(self.backupIO); DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); @@ -379,7 +379,7 @@ NS_ASSUME_NONNULL_BEGIN return NO; } - OWSDBExportStream *exportStream = [[OWSDBExportStream alloc] initWithEncryption:self.encryption]; + OWSDBExportStream *exportStream = [[OWSDBExportStream alloc] initWithBackupIO:self.backupIO]; __block BOOL aborted = NO; typedef BOOL (^EntityFilter)(id object); @@ -453,9 +453,9 @@ NS_ASSUME_NONNULL_BEGIN // OWSAttachmentExport is used to lazily write an encrypted copy of the // attachment to disk. OWSAttachmentExport *attachmentExport = - [[OWSAttachmentExport alloc] initWithEncryption:self.encryption - attachmentId:attachmentStream.uniqueId - attachmentFilePath:filePath]; + [[OWSAttachmentExport alloc] initWithBackupIO:self.backupIO + attachmentId:attachmentStream.uniqueId + attachmentFilePath:filePath]; [self.unsavedAttachmentExports addObject:attachmentExport]; return YES; @@ -727,7 +727,7 @@ NS_ASSUME_NONNULL_BEGIN OWSAssert(self.savedDatabaseItems.count > 0); OWSAssert(self.savedAttachmentItems); OWSAssert(self.jobTempDirPath.length > 0); - OWSAssert(self.encryption); + OWSAssert(self.backupIO); NSDictionary *json = @{ kOWSBackup_ManifestKey_DatabaseFiles : [self jsonForItems:self.savedDatabaseItems], @@ -743,7 +743,7 @@ NS_ASSUME_NONNULL_BEGIN OWSProdLogAndFail(@"%@ error encoding manifest file: %@", self.logTag, error); return nil; } - return [self.encryption encryptDataAsTempFile:jsonData encryptionKey:self.delegate.backupEncryptionKey]; + return [self.backupIO encryptDataAsTempFile:jsonData encryptionKey:self.delegate.backupEncryptionKey]; } - (NSArray *> *)jsonForItems:(NSArray *)items diff --git a/Signal/src/util/OWSBackupEncryption.h b/Signal/src/util/OWSBackupIO.h similarity index 97% rename from Signal/src/util/OWSBackupEncryption.h rename to Signal/src/util/OWSBackupIO.h index 38a8a12814..38414fa06f 100644 --- a/Signal/src/util/OWSBackupEncryption.h +++ b/Signal/src/util/OWSBackupIO.h @@ -14,7 +14,7 @@ NS_ASSUME_NONNULL_BEGIN #pragma mark - -@interface OWSBackupEncryption : NSObject +@interface OWSBackupIO : NSObject - (instancetype)init NS_UNAVAILABLE; diff --git a/Signal/src/util/OWSBackupEncryption.m b/Signal/src/util/OWSBackupIO.m similarity index 98% rename from Signal/src/util/OWSBackupEncryption.m rename to Signal/src/util/OWSBackupIO.m index c618a7e3e6..8b89c2f3af 100644 --- a/Signal/src/util/OWSBackupEncryption.m +++ b/Signal/src/util/OWSBackupIO.m @@ -2,7 +2,7 @@ // Copyright (c) 2018 Open Whisper Systems. All rights reserved. // -#import "OWSBackupEncryption.h" +#import "OWSBackupIO.h" #import #import @@ -19,7 +19,7 @@ static const NSUInteger kOWSBackupKeyLength = 32; #pragma mark - -@interface OWSBackupEncryption () +@interface OWSBackupIO () @property (nonatomic) NSString *jobTempDirPath; @@ -27,7 +27,7 @@ static const NSUInteger kOWSBackupKeyLength = 32; #pragma mark - -@implementation OWSBackupEncryption +@implementation OWSBackupIO - (instancetype)initWithJobTempDirPath:(NSString *)jobTempDirPath { diff --git a/Signal/src/util/OWSBackupImportJob.m b/Signal/src/util/OWSBackupImportJob.m index a792d7f4f8..659bc86ab8 100644 --- a/Signal/src/util/OWSBackupImportJob.m +++ b/Signal/src/util/OWSBackupImportJob.m @@ -3,7 +3,7 @@ // #import "OWSBackupImportJob.h" -#import "OWSBackupEncryption.h" +#import "OWSBackupIO.h" #import "OWSDatabaseMigration.h" #import "OWSDatabaseMigrationRunner.h" #import "Signal-Swift.h" @@ -46,7 +46,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe @property (nonatomic, nullable) OWSBackgroundTask *backgroundTask; -@property (nonatomic) OWSBackupEncryption *encryption; +@property (nonatomic) OWSBackupIO *backupIO; @property (nonatomic) NSArray *databaseItems; @property (nonatomic) NSArray *attachmentsItems; @@ -173,7 +173,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return NO; } - self.encryption = [[OWSBackupEncryption alloc] initWithJobTempDirPath:self.jobTempDirPath]; + self.backupIO = [[OWSBackupIO alloc] initWithJobTempDirPath:self.jobTempDirPath]; return YES; } @@ -209,7 +209,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe - (void)processManifest:(NSData *)manifestDataEncrypted completion:(OWSBackupJobBoolCompletion)completion { OWSAssert(completion); - OWSAssert(self.encryption); + OWSAssert(self.backupIO); if (self.isComplete) { return; @@ -218,7 +218,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); NSData *_Nullable manifestDataDecrypted = - [self.encryption decryptDataAsData:manifestDataEncrypted encryptionKey:self.delegate.backupEncryptionKey]; + [self.backupIO decryptDataAsData:manifestDataEncrypted encryptionKey:self.delegate.backupEncryptionKey]; if (!manifestDataDecrypted) { OWSProdLogAndFail(@"%@ Could not decrypt manifest.", self.logTag); return completion(NO); @@ -408,9 +408,9 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe DDLogError(@"%@ skipping redundant file restore: %@.", self.logTag, dstFilePath); continue; } - if (![self.encryption decryptFileAsFile:item.downloadFilePath - dstFilePath:dstFilePath - encryptionKey:item.encryptionKey]) { + if (![self.backupIO decryptFileAsFile:item.downloadFilePath + dstFilePath:dstFilePath + encryptionKey:item.encryptionKey]) { DDLogError(@"%@ attachment could not be restored.", self.logTag); // Attachment-related errors are recoverable and can be ignored. continue; @@ -498,15 +498,15 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe progress:@(count / (CGFloat)self.databaseItems.count)]; NSData *_Nullable compressedData = - [self.encryption decryptFileAsData:item.downloadFilePath encryptionKey:item.encryptionKey]; + [self.backupIO decryptFileAsData:item.downloadFilePath encryptionKey:item.encryptionKey]; if (!compressedData) { // Database-related errors are unrecoverable. aborted = YES; return completion(NO); } NSData *_Nullable uncompressedData = - [self.encryption decompressData:compressedData - uncompressedDataLength:item.uncompressedDataLength.unsignedIntValue]; + [self.backupIO decompressData:compressedData + uncompressedDataLength:item.uncompressedDataLength.unsignedIntValue]; if (!uncompressedData) { // Database-related errors are unrecoverable. aborted = YES; diff --git a/Signal/src/util/OWSBackupJob.m b/Signal/src/util/OWSBackupJob.m index a0b9c933ce..04a35b4308 100644 --- a/Signal/src/util/OWSBackupJob.m +++ b/Signal/src/util/OWSBackupJob.m @@ -3,7 +3,6 @@ // #import "OWSBackupJob.h" -#import "OWSBackupEncryption.h" #import "Signal-Swift.h" #import #import From 2ebd8668b42a895b2063d2ba4ddb6382db48d192 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Fri, 16 Mar 2018 14:58:18 -0300 Subject: [PATCH 05/13] Fix bugs in new db representation, add batch record deletion, improve memory management. --- .../AppSettings/AppSettingsViewController.m | 5 - .../src/ViewControllers/HomeViewController.m | 4 - Signal/src/util/OWSBackupAPI.swift | 37 +-- Signal/src/util/OWSBackupExportJob.m | 87 ++++--- Signal/src/util/OWSBackupIO.m | 228 ++++++++++-------- Signal/src/util/OWSBackupImportJob.m | 108 +++++---- 6 files changed, 254 insertions(+), 215 deletions(-) diff --git a/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m b/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m index 43e23fec77..26f94e5e97 100644 --- a/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m +++ b/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m @@ -90,11 +90,6 @@ self.title = NSLocalizedString(@"SETTINGS_NAV_BAR_TITLE", @"Title for settings activity"); [self updateTableContents]; - - dispatch_async(dispatch_get_main_queue(), ^{ - [self showBackup]; - // [self showDebugUI]; - }); } - (void)viewWillAppear:(BOOL)animated diff --git a/Signal/src/ViewControllers/HomeViewController.m b/Signal/src/ViewControllers/HomeViewController.m index 8d47def8b3..dccd63e158 100644 --- a/Signal/src/ViewControllers/HomeViewController.m +++ b/Signal/src/ViewControllers/HomeViewController.m @@ -426,10 +426,6 @@ typedef NS_ENUM(NSInteger, CellState) { kArchiveState, kInboxState }; } [self checkIfEmptyView]; - - dispatch_async(dispatch_get_main_queue(), ^{ - [self settingsButtonPressed:nil]; - }); } - (void)viewWillDisappear:(BOOL)animated diff --git a/Signal/src/util/OWSBackupAPI.swift b/Signal/src/util/OWSBackupAPI.swift index 79344a34e1..ad67f410e3 100644 --- a/Signal/src/util/OWSBackupAPI.swift +++ b/Signal/src/util/OWSBackupAPI.swift @@ -225,28 +225,30 @@ import CloudKit // MARK: - Delete @objc - public class func deleteRecordFromCloud(recordName: String, + public class func deleteRecordsFromCloud(recordNames: [String], success: @escaping (()) -> Void, failure: @escaping (Error) -> Void) { - deleteRecordFromCloud(recordName: recordName, + deleteRecordsFromCloud(recordNames: recordNames, remainingRetries: maxRetries, success: success, failure: failure) } - private class func deleteRecordFromCloud(recordName: String, + private class func deleteRecordsFromCloud(recordNames: [String], remainingRetries: Int, success: @escaping (()) -> Void, failure: @escaping (Error) -> Void) { - let recordID = CKRecordID(recordName: recordName) - - database().delete(withRecordID: recordID) { - (_, error) in + var recordIDs = [CKRecordID]() + for recordName in recordNames { + recordIDs.append(CKRecordID(recordName: recordName)) + } + let deleteOperation = CKModifyRecordsOperation(recordsToSave: nil, recordIDsToDelete: recordIDs) + deleteOperation.modifyRecordsCompletionBlock = { (records, recordIds, error) in let outcome = outcomeForCloudKitError(error: error, - remainingRetries: remainingRetries, - label: "Delete Record") + remainingRetries: remainingRetries, + label: "Delete Records") switch outcome { case .success: success() @@ -254,23 +256,24 @@ import CloudKit failure(outcomeError) case .failureRetryAfterDelay(let retryDelay): DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: { - deleteRecordFromCloud(recordName: recordName, - remainingRetries: remainingRetries - 1, - success: success, - failure: failure) + deleteRecordsFromCloud(recordNames: recordNames, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) }) case .failureRetryWithoutDelay: DispatchQueue.global().async { - deleteRecordFromCloud(recordName: recordName, - remainingRetries: remainingRetries - 1, - success: success, - failure: failure) + deleteRecordsFromCloud(recordNames: recordNames, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) } case .unknownItem: owsFail("\(self.logTag) unexpected CloudKit response.") failure(invalidServiceResponseError()) } } + database().add(deleteOperation) } // MARK: - Exists? diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index aba34796a0..ce2bcecded 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -124,7 +124,9 @@ NS_ASSUME_NONNULL_BEGIN static const int kMaxDBSnapshotSize = 1000; if (self.cachedItemCount > kMaxDBSnapshotSize) { - return [self flush]; + @autoreleasepool { + return [self flush]; + } } return YES; @@ -137,27 +139,31 @@ NS_ASSUME_NONNULL_BEGIN return YES; } - NSData *_Nullable uncompressedData = [self.backupSnapshotBuilder build].data; - NSUInteger uncompressedDataLength = uncompressedData.length; - self.backupSnapshotBuilder = nil; - self.cachedItemCount = 0; - if (!uncompressedData) { - OWSProdLogAndFail(@"%@ couldn't convert database snapshot to data.", self.logTag); - return NO; + // Try to release allocated buffers ASAP. + @autoreleasepool { + + NSData *_Nullable uncompressedData = [self.backupSnapshotBuilder build].data; + NSUInteger uncompressedDataLength = uncompressedData.length; + self.backupSnapshotBuilder = nil; + self.cachedItemCount = 0; + if (!uncompressedData) { + OWSProdLogAndFail(@"%@ couldn't convert database snapshot to data.", self.logTag); + return NO; + } + + NSData *compressedData = [self.backupIO compressData:uncompressedData]; + + OWSBackupEncryptedItem *_Nullable encryptedItem = [self.backupIO encryptDataAsTempFile:compressedData]; + if (!encryptedItem) { + OWSProdLogAndFail(@"%@ couldn't encrypt database snapshot.", self.logTag); + return NO; + } + + OWSBackupExportItem *exportItem = [[OWSBackupExportItem alloc] initWithEncryptedItem:encryptedItem]; + exportItem.uncompressedDataLength = @(uncompressedDataLength); + [self.exportItems addObject:exportItem]; } - NSData *compressedData = [self.backupIO compressData:uncompressedData]; - - OWSBackupEncryptedItem *_Nullable encryptedItem = [self.backupIO encryptDataAsTempFile:compressedData]; - if (!encryptedItem) { - OWSProdLogAndFail(@"%@ couldn't encrypt database snapshot.", self.logTag); - return NO; - } - - OWSBackupExportItem *exportItem = [[OWSBackupExportItem alloc] initWithEncryptedItem:encryptedItem]; - exportItem.uncompressedDataLength = @(uncompressedDataLength); - [self.exportItems addObject:exportItem]; - return YES; } @@ -500,9 +506,11 @@ NS_ASSUME_NONNULL_BEGIN return NO; } - if (![exportStream flush]) { - OWSProdLogAndFail(@"%@ Could not flush database snapshots.", self.logTag); - return NO; + @autoreleasepool { + if (![exportStream flush]) { + OWSProdLogAndFail(@"%@ Could not flush database snapshots.", self.logTag); + return NO; + } } self.unsavedDatabaseItems = [exportStream.exportItems mutableCopy]; @@ -629,15 +637,17 @@ NS_ASSUME_NONNULL_BEGIN OWSAttachmentExport *attachmentExport = self.unsavedAttachmentExports.lastObject; [self.unsavedAttachmentExports removeLastObject]; - // OWSAttachmentExport is used to lazily write an encrypted copy of the - // attachment to disk. - if (![attachmentExport prepareForUpload]) { - // Attachment files are non-critical so any error uploading them is recoverable. - [weakSelf saveNextFileToCloud:completion]; - return YES; + @autoreleasepool { + // OWSAttachmentExport is used to lazily write an encrypted copy of the + // attachment to disk. + if (![attachmentExport prepareForUpload]) { + // Attachment files are non-critical so any error uploading them is recoverable. + [weakSelf saveNextFileToCloud:completion]; + return YES; + } + OWSAssert(attachmentExport.relativeFilePath.length > 0); + OWSAssert(attachmentExport.encryptedItem); } - OWSAssert(attachmentExport.relativeFilePath.length > 0); - OWSAssert(attachmentExport.encryptedItem); [OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentExport.attachmentId fileUrlBlock:^{ @@ -856,16 +866,21 @@ NS_ASSUME_NONNULL_BEGIN @"Indicates that the cloud is being cleaned up.") progress:@(progress)]; - NSString *recordName = obsoleteRecordNames.lastObject; - [obsoleteRecordNames removeLastObject]; + static const NSUInteger kMaxBatchSize = 100; + NSMutableArray *batchRecordNames = [NSMutableArray new]; + while (obsoleteRecordNames.count > 0 && batchRecordNames.count < kMaxBatchSize) { + NSString *recordName = obsoleteRecordNames.lastObject; + [obsoleteRecordNames removeLastObject]; + [batchRecordNames addObject:recordName]; + } __weak OWSBackupExportJob *weakSelf = self; - [OWSBackupAPI deleteRecordFromCloudWithRecordName:recordName + [OWSBackupAPI deleteRecordsFromCloudWithRecordNames:batchRecordNames success:^{ // Ensure that we continue to work off the main thread. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ [weakSelf deleteRecordsFromCloud:obsoleteRecordNames - deletedCount:deletedCount + 1 + deletedCount:deletedCount + batchRecordNames.count completion:completion]; }); } @@ -874,7 +889,7 @@ NS_ASSUME_NONNULL_BEGIN dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ // Cloud cleanup is non-critical so any error is recoverable. [weakSelf deleteRecordsFromCloud:obsoleteRecordNames - deletedCount:deletedCount + 1 + deletedCount:deletedCount + batchRecordNames.count completion:completion]; }); }]; diff --git a/Signal/src/util/OWSBackupIO.m b/Signal/src/util/OWSBackupIO.m index 8b89c2f3af..ebf8f59858 100644 --- a/Signal/src/util/OWSBackupIO.m +++ b/Signal/src/util/OWSBackupIO.m @@ -56,13 +56,16 @@ static const NSUInteger kOWSBackupKeyLength = 32; OWSAssert(srcFilePath.length > 0); OWSAssert(encryptionKey.length > 0); - // TODO: Encrypt the file without loading it into memory. - NSData *_Nullable srcData = [NSData dataWithContentsOfFile:srcFilePath]; - if (!srcData) { - OWSProdLogAndFail(@"%@ could not load file into memory", self.logTag); - return nil; + @autoreleasepool { + + // TODO: Encrypt the file without loading it into memory. + NSData *_Nullable srcData = [NSData dataWithContentsOfFile:srcFilePath]; + if (srcData.length < 1) { + OWSProdLogAndFail(@"%@ could not load file into memory for encryption.", self.logTag); + return nil; + } + return [self encryptDataAsTempFile:srcData encryptionKey:encryptionKey]; } - return [self encryptDataAsTempFile:srcData encryptionKey:encryptionKey]; } - (nullable OWSBackupEncryptedItem *)encryptDataAsTempFile:(NSData *)srcData @@ -74,25 +77,30 @@ static const NSUInteger kOWSBackupKeyLength = 32; return [self encryptDataAsTempFile:srcData encryptionKey:encryptionKey]; } -- (nullable OWSBackupEncryptedItem *)encryptDataAsTempFile:(NSData *)srcData encryptionKey:(NSData *)encryptionKey +- (nullable OWSBackupEncryptedItem *)encryptDataAsTempFile:(NSData *)unencryptedData + encryptionKey:(NSData *)encryptionKey { - OWSAssert(srcData); + OWSAssert(unencryptedData); OWSAssert(encryptionKey.length > 0); - // TODO: Encrypt the data using key; + @autoreleasepool { - NSString *dstFilePath = [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; - NSError *error; - BOOL success = [srcData writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; - if (!success || error) { - OWSProdLogAndFail(@"%@ error writing encrypted data: %@", self.logTag, error); - return nil; + // TODO: Encrypt the data using key; + NSData *encryptedData = unencryptedData; + + NSString *dstFilePath = [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; + NSError *error; + BOOL success = [encryptedData writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; + if (!success || error) { + OWSProdLogAndFail(@"%@ error writing encrypted data: %@", self.logTag, error); + return nil; + } + [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; + OWSBackupEncryptedItem *item = [OWSBackupEncryptedItem new]; + item.filePath = dstFilePath; + item.encryptionKey = encryptionKey; + return item; } - [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; - OWSBackupEncryptedItem *item = [OWSBackupEncryptedItem new]; - item.filePath = dstFilePath; - item.encryptionKey = encryptionKey; - return item; } #pragma mark - Decrypt @@ -104,22 +112,25 @@ static const NSUInteger kOWSBackupKeyLength = 32; OWSAssert(srcFilePath.length > 0); OWSAssert(encryptionKey.length > 0); - // TODO: Decrypt the file without loading it into memory. - NSData *data = [self decryptFileAsData:srcFilePath encryptionKey:encryptionKey]; + @autoreleasepool { - if (!data) { - return NO; + // TODO: Decrypt the file without loading it into memory. + NSData *data = [self decryptFileAsData:srcFilePath encryptionKey:encryptionKey]; + + if (data.length < 1) { + return NO; + } + + NSError *error; + BOOL success = [data writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; + if (!success || error) { + OWSProdLogAndFail(@"%@ error writing decrypted data: %@", self.logTag, error); + return NO; + } + [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; + + return YES; } - - NSError *error; - BOOL success = [data writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; - if (!success || error) { - OWSProdLogAndFail(@"%@ error writing decrypted data: %@", self.logTag, error); - return NO; - } - [OWSFileSystem protectFileOrFolderAtPath:dstFilePath]; - - return YES; } - (nullable NSData *)decryptFileAsData:(NSString *)srcFilePath encryptionKey:(NSData *)encryptionKey @@ -127,30 +138,36 @@ static const NSUInteger kOWSBackupKeyLength = 32; OWSAssert(srcFilePath.length > 0); OWSAssert(encryptionKey.length > 0); - if (![NSFileManager.defaultManager fileExistsAtPath:srcFilePath]) { - DDLogError(@"%@ missing downloaded file.", self.logTag); - return nil; - } + @autoreleasepool { - // TODO: Decrypt the file without loading it into memory. - NSData *_Nullable srcData = [NSData dataWithContentsOfFile:srcFilePath]; - if (!srcData) { - OWSProdLogAndFail(@"%@ could not load file into memory", self.logTag); - return nil; - } + if (![NSFileManager.defaultManager fileExistsAtPath:srcFilePath]) { + DDLogError(@"%@ missing downloaded file.", self.logTag); + return nil; + } - NSData *_Nullable dstData = [self decryptDataAsData:srcData encryptionKey:encryptionKey]; - return dstData; + NSData *_Nullable srcData = [NSData dataWithContentsOfFile:srcFilePath]; + if (srcData.length < 1) { + OWSProdLogAndFail(@"%@ could not load file into memory for decryption.", self.logTag); + return nil; + } + + NSData *_Nullable dstData = [self decryptDataAsData:srcData encryptionKey:encryptionKey]; + return dstData; + } } -- (nullable NSData *)decryptDataAsData:(NSData *)srcData encryptionKey:(NSData *)encryptionKey +- (nullable NSData *)decryptDataAsData:(NSData *)encryptedData encryptionKey:(NSData *)encryptionKey { - OWSAssert(srcData); + OWSAssert(encryptedData); OWSAssert(encryptionKey.length > 0); - // TODO: Decrypt the data using key; + @autoreleasepool { - return srcData; + // TODO: Decrypt the data using key; + NSData *unencryptedData = encryptedData; + + return unencryptedData; + } } #pragma mark - Compression @@ -159,67 +176,76 @@ static const NSUInteger kOWSBackupKeyLength = 32; { OWSAssert(srcData); - if (!srcData) { - OWSProdLogAndFail(@"%@ missing unencrypted data.", self.logTag); - return nil; - } + @autoreleasepool { - size_t srcLength = [srcData length]; - const uint8_t *srcBuffer = (const uint8_t *)[srcData bytes]; - if (!srcBuffer) { - return nil; - } - // This assumes that dst will always be smaller than src. - // - // We slightly pad the buffer size to account for the worst case. - uint8_t *dstBuffer = malloc(sizeof(uint8_t) * srcLength) + 64 * 1024; - if (!dstBuffer) { - return nil; - } - // TODO: Should we use COMPRESSION_LZFSE? - size_t dstLength = compression_encode_buffer(dstBuffer, srcLength, srcBuffer, srcLength, NULL, COMPRESSION_LZFSE); - NSData *compressedData = [NSData dataWithBytes:dstBuffer length:dstLength]; - DDLogVerbose(@"%@ compressed %zd -> %zd = %0.2f", - self.logTag, - srcLength, - dstLength, - (srcLength > 0 ? (dstLength / (CGFloat)srcLength) : 0)); - free(dstBuffer); + if (!srcData) { + OWSProdLogAndFail(@"%@ missing unencrypted data.", self.logTag); + return nil; + } - return compressedData; + size_t srcLength = [srcData length]; + const uint8_t *srcBuffer = (const uint8_t *)[srcData bytes]; + if (!srcBuffer) { + return nil; + } + // This assumes that dst will always be smaller than src. + // + // We slightly pad the buffer size to account for the worst case. + size_t dstBufferLength = srcLength + 64 * 1024; + uint8_t *dstBuffer = malloc(sizeof(uint8_t) * dstBufferLength); + if (!dstBuffer) { + return nil; + } + // TODO: Should we use COMPRESSION_LZFSE? + size_t dstLength + = compression_encode_buffer(dstBuffer, dstBufferLength, srcBuffer, srcLength, NULL, COMPRESSION_LZFSE); + NSData *compressedData = [NSData dataWithBytesNoCopy:dstBuffer length:dstLength freeWhenDone:YES]; + + DDLogVerbose(@"%@ compressed %zd -> %zd = %0.2f", + self.logTag, + srcLength, + dstLength, + (srcLength > 0 ? (dstLength / (CGFloat)srcLength) : 0)); + + return compressedData; + } } - (nullable NSData *)decompressData:(NSData *)srcData uncompressedDataLength:(NSUInteger)uncompressedDataLength { OWSAssert(srcData); - if (!srcData) { - OWSProdLogAndFail(@"%@ missing unencrypted data.", self.logTag); - return nil; - } + @autoreleasepool { - size_t srcLength = [srcData length]; - const uint8_t *srcBuffer = (const uint8_t *)[srcData bytes]; - if (!srcBuffer) { - return nil; - } - // We pad the buffer to be defensive. - uint8_t *dstBuffer = malloc(sizeof(uint8_t) * (uncompressedDataLength + 1024)); - if (!dstBuffer) { - return nil; - } - // TODO: Should we use COMPRESSION_LZFSE? - size_t dstLength = compression_decode_buffer(dstBuffer, srcLength, srcBuffer, srcLength, NULL, COMPRESSION_LZFSE); - NSData *decompressedData = [NSData dataWithBytes:dstBuffer length:dstLength]; - OWSAssert(decompressedData.length == uncompressedDataLength); - DDLogVerbose(@"%@ decompressed %zd -> %zd = %0.2f", - self.logTag, - srcLength, - dstLength, - (dstLength > 0 ? (srcLength / (CGFloat)dstLength) : 0)); - free(dstBuffer); + if (!srcData) { + OWSProdLogAndFail(@"%@ missing unencrypted data.", self.logTag); + return nil; + } - return decompressedData; + size_t srcLength = [srcData length]; + const uint8_t *srcBuffer = (const uint8_t *)[srcData bytes]; + if (!srcBuffer) { + return nil; + } + // We pad the buffer to be defensive. + size_t dstBufferLength = uncompressedDataLength + 1024; + uint8_t *dstBuffer = malloc(sizeof(uint8_t) * dstBufferLength); + if (!dstBuffer) { + return nil; + } + // TODO: Should we use COMPRESSION_LZFSE? + size_t dstLength + = compression_decode_buffer(dstBuffer, dstBufferLength, srcBuffer, srcLength, NULL, COMPRESSION_LZFSE); + NSData *decompressedData = [NSData dataWithBytesNoCopy:dstBuffer length:dstLength freeWhenDone:YES]; + OWSAssert(decompressedData.length == uncompressedDataLength); + DDLogVerbose(@"%@ decompressed %zd -> %zd = %0.2f", + self.logTag, + srcLength, + dstLength, + (dstLength > 0 ? (srcLength / (CGFloat)dstLength) : 0)); + + return decompressedData; + } } @end diff --git a/Signal/src/util/OWSBackupImportJob.m b/Signal/src/util/OWSBackupImportJob.m index 659bc86ab8..78b2859681 100644 --- a/Signal/src/util/OWSBackupImportJob.m +++ b/Signal/src/util/OWSBackupImportJob.m @@ -256,7 +256,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe OWSAssert(json); OWSAssert(key.length); - if (![json isKindOfClass:[NSArray class]]) { + if (![json isKindOfClass:[NSDictionary class]]) { OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key); return nil; } @@ -408,12 +408,14 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe DDLogError(@"%@ skipping redundant file restore: %@.", self.logTag, dstFilePath); continue; } - if (![self.backupIO decryptFileAsFile:item.downloadFilePath - dstFilePath:dstFilePath - encryptionKey:item.encryptionKey]) { - DDLogError(@"%@ attachment could not be restored.", self.logTag); - // Attachment-related errors are recoverable and can be ignored. - continue; + @autoreleasepool { + if (![self.backupIO decryptFileAsFile:item.downloadFilePath + dstFilePath:dstFilePath + encryptionKey:item.encryptionKey]) { + DDLogError(@"%@ attachment could not be restored.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + continue; + } } DDLogError(@"%@ restored file: %@.", self.logTag, item.relativeFilePath); @@ -497,60 +499,62 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe @"Indicates that the backup database is being restored.") progress:@(count / (CGFloat)self.databaseItems.count)]; - NSData *_Nullable compressedData = - [self.backupIO decryptFileAsData:item.downloadFilePath encryptionKey:item.encryptionKey]; - if (!compressedData) { - // Database-related errors are unrecoverable. - aborted = YES; - return completion(NO); - } - NSData *_Nullable uncompressedData = - [self.backupIO decompressData:compressedData - uncompressedDataLength:item.uncompressedDataLength.unsignedIntValue]; - if (!uncompressedData) { - // Database-related errors are unrecoverable. - aborted = YES; - return completion(NO); - } - OWSSignalServiceProtosBackupSnapshot *_Nullable entities = - [OWSSignalServiceProtosBackupSnapshot parseFromData:uncompressedData]; - if (!entities || entities.entity.count < 1) { - DDLogError(@"%@ missing entities.", self.logTag); - // Database-related errors are unrecoverable. - aborted = YES; - return completion(NO); - } - for (OWSSignalServiceProtosBackupSnapshotBackupEntity *entity in entities.entity) { - NSData *_Nullable entityData = entity.entityData; - if (entityData.length < 1) { - DDLogError(@"%@ missing entity data.", self.logTag); + @autoreleasepool { + NSData *_Nullable compressedData = + [self.backupIO decryptFileAsData:item.downloadFilePath encryptionKey:item.encryptionKey]; + if (!compressedData) { // Database-related errors are unrecoverable. aborted = YES; return completion(NO); } - - __block TSYapDatabaseObject *object = nil; - @try { - NSKeyedUnarchiver *unarchiver = [[NSKeyedUnarchiver alloc] initForReadingWithData:entityData]; - object = [unarchiver decodeObjectForKey:@"root"]; - if (![object isKindOfClass:[object class]]) { - DDLogError(@"%@ invalid decoded entity: %@.", self.logTag, [object class]); + NSData *_Nullable uncompressedData = + [self.backupIO decompressData:compressedData + uncompressedDataLength:item.uncompressedDataLength.unsignedIntValue]; + if (!uncompressedData) { + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + OWSSignalServiceProtosBackupSnapshot *_Nullable entities = + [OWSSignalServiceProtosBackupSnapshot parseFromData:uncompressedData]; + if (!entities || entities.entity.count < 1) { + DDLogError(@"%@ missing entities.", self.logTag); + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + for (OWSSignalServiceProtosBackupSnapshotBackupEntity *entity in entities.entity) { + NSData *_Nullable entityData = entity.entityData; + if (entityData.length < 1) { + DDLogError(@"%@ missing entity data.", self.logTag); // Database-related errors are unrecoverable. aborted = YES; return completion(NO); } - } @catch (NSException *exception) { - DDLogError(@"%@ could not decode entity.", self.logTag); - // Database-related errors are unrecoverable. - aborted = YES; - return completion(NO); - } - [object saveWithTransaction:transaction]; - copiedEntities++; - NSString *collection = [object.class collection]; - NSUInteger restoredEntityCount = restoredEntityCounts[collection].unsignedIntValue; - restoredEntityCounts[collection] = @(restoredEntityCount + 1); + __block TSYapDatabaseObject *object = nil; + @try { + NSKeyedUnarchiver *unarchiver = [[NSKeyedUnarchiver alloc] initForReadingWithData:entityData]; + object = [unarchiver decodeObjectForKey:@"root"]; + if (![object isKindOfClass:[object class]]) { + DDLogError(@"%@ invalid decoded entity: %@.", self.logTag, [object class]); + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + } @catch (NSException *exception) { + DDLogError(@"%@ could not decode entity.", self.logTag); + // Database-related errors are unrecoverable. + aborted = YES; + return completion(NO); + } + + [object saveWithTransaction:transaction]; + copiedEntities++; + NSString *collection = [object.class collection]; + NSUInteger restoredEntityCount = restoredEntityCounts[collection].unsignedIntValue; + restoredEntityCounts[collection] = @(restoredEntityCount + 1); + } } } }]; From 610bbacd218f424c1544bfaaabd73efcf64f9e2f Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Sat, 17 Mar 2018 09:31:27 -0300 Subject: [PATCH 06/13] Clean up ahead of PR. --- Pods | 2 +- SignalServiceKit/protobuf/Makefile | 6 +- .../src/Storage/OWSBackupStorage.h | 30 ---- .../src/Storage/OWSBackupStorage.m | 153 ------------------ 4 files changed, 4 insertions(+), 187 deletions(-) delete mode 100644 SignalServiceKit/src/Storage/OWSBackupStorage.h delete mode 100644 SignalServiceKit/src/Storage/OWSBackupStorage.m diff --git a/Pods b/Pods index 02d9e07b52..39c23000fa 160000 --- a/Pods +++ b/Pods @@ -1 +1 @@ -Subproject commit 02d9e07b52a3c3a43cc29a90c586b61fc30cd032 +Subproject commit 39c23000fa4c377adddcd952b8a8093a14ade407 diff --git a/SignalServiceKit/protobuf/Makefile b/SignalServiceKit/protobuf/Makefile index 42772b7db3..d7b4c47b7c 100644 --- a/SignalServiceKit/protobuf/Makefile +++ b/SignalServiceKit/protobuf/Makefile @@ -1,9 +1,9 @@ # See README.md in this dir for prerequisite setup. - + PROTOC=protoc \ --plugin=/usr/local/bin/protoc-gen-objc \ - --proto_path="${HOME}/code/workspace/ows/protobuf-objc/src/compiler/" \ - --proto_path="${HOME}/code/workspace/ows/protobuf-objc/src/compiler/google/protobuf/" \ + --proto_path="${HOME}/src/WhisperSystems/protobuf-objc/src/compiler/" \ + --proto_path="${HOME}/src/WhisperSystems/protobuf-objc/src/compiler/google/protobuf/" \ --proto_path='./' all: signal_service_proto provisioning_protos fingerprint_protos diff --git a/SignalServiceKit/src/Storage/OWSBackupStorage.h b/SignalServiceKit/src/Storage/OWSBackupStorage.h deleted file mode 100644 index babb381714..0000000000 --- a/SignalServiceKit/src/Storage/OWSBackupStorage.h +++ /dev/null @@ -1,30 +0,0 @@ -// -// Copyright (c) 2018 Open Whisper Systems. All rights reserved. -// - -// -// Copyright (c) 2018 Open Whisper Systems. All rights reserved. -// -#import "OWSPrimaryStorage.h" -#import "OWSStorage.h" - -NS_ASSUME_NONNULL_BEGIN - -typedef NSData *_Nullable (^BackupStorageKeySpecBlock)(void); - -@interface OWSBackupStorage : OWSStorage - -- (instancetype)init NS_UNAVAILABLE; - -- (instancetype)initStorage NS_UNAVAILABLE; - -- (instancetype)initBackupStorageWithDatabaseDirPath:(NSString *)databaseDirPath - keySpecBlock:(BackupStorageKeySpecBlock)keySpecBlock NS_DESIGNATED_INITIALIZER; - -- (void)runSyncRegistrations; -- (void)runAsyncRegistrationsWithCompletion:(void (^_Nonnull)(void))completion; -- (BOOL)areAllRegistrationsComplete; - -@end - -NS_ASSUME_NONNULL_END diff --git a/SignalServiceKit/src/Storage/OWSBackupStorage.m b/SignalServiceKit/src/Storage/OWSBackupStorage.m deleted file mode 100644 index 38200e543c..0000000000 --- a/SignalServiceKit/src/Storage/OWSBackupStorage.m +++ /dev/null @@ -1,153 +0,0 @@ -// -// Copyright (c) 2018 Open Whisper Systems. All rights reserved. -// - -#import "OWSBackupStorage.h" -#import "OWSFileSystem.h" -#import "OWSStorage+Subclass.h" - -NS_ASSUME_NONNULL_BEGIN - -@interface OWSBackupStorage () - -@property (atomic) BOOL areAsyncRegistrationsComplete; -@property (atomic) BOOL areSyncRegistrationsComplete; - -@property (nonatomic, readonly) NSString *databaseDirPath; -@property (nonatomic, readonly) BackupStorageKeySpecBlock keySpecBlock; - -@end - -#pragma mark - - -@implementation OWSBackupStorage - -- (instancetype)initBackupStorageWithDatabaseDirPath:(NSString *)databaseDirPath - keySpecBlock:(BackupStorageKeySpecBlock)keySpecBlock -{ - OWSAssert(databaseDirPath.length > 0); - OWSAssert(keySpecBlock); - OWSAssert([OWSFileSystem ensureDirectoryExists:databaseDirPath]); - - self = [super initStorage]; - - if (self) { - _databaseDirPath = databaseDirPath; - _keySpecBlock = keySpecBlock; - - [self loadDatabase]; - } - - return self; -} - -- (void)loadDatabase -{ - [super loadDatabase]; - - [self protectFiles]; -} - -- (void)resetStorage -{ - [super resetStorage]; -} - -- (void)runSyncRegistrations -{ - runSyncRegistrationsForStorage(self); - - // See comments on OWSDatabaseConnection. - // - // In the absence of finding documentation that can shed light on the issue we've been - // seeing, this issue only seems to affect sync and not async registrations. We've always - // been opening write transactions before the async registrations complete without negative - // consequences. - OWSAssert(!self.areSyncRegistrationsComplete); - self.areSyncRegistrationsComplete = YES; -} - -- (void)runAsyncRegistrationsWithCompletion:(void (^_Nonnull)(void))completion -{ - OWSAssert(completion); - - runAsyncRegistrationsForStorage(self); - - DDLogVerbose(@"%@ async registrations enqueued.", self.logTag); - - // Block until all async registrations are complete. - // - // NOTE: This has to happen on the "registration connection" for this - // database. - YapDatabaseConnection *dbConnection = self.registrationConnection; - OWSAssert(self.registrationConnection); - [dbConnection flushTransactionsWithCompletionQueue:dispatch_get_main_queue() - completionBlock:^{ - OWSAssert(!self.areAsyncRegistrationsComplete); - - DDLogVerbose(@"%@ async registrations complete.", self.logTag); - - self.areAsyncRegistrationsComplete = YES; - - completion(); - }]; -} - -- (void)protectFiles -{ - [self logFileSizes]; - - // Protect the entire new database directory. - [OWSFileSystem protectFileOrFolderAtPath:self.databaseDirPath]; -} - -+ (NSString *)databaseFilename -{ - return @"SignalBackup.sqlite"; -} - -- (NSString *)databaseFilename -{ - return OWSBackupStorage.databaseFilename; -} - -- (NSString *)databaseFilename_SHM -{ - return [self.databaseFilename stringByAppendingString:@"-shm"]; -} - -- (NSString *)databaseFilename_WAL -{ - return [self.databaseFilename stringByAppendingString:@"-wal"]; -} - -- (NSString *)databaseFilePath -{ - return [self.databaseDirPath stringByAppendingPathComponent:self.databaseFilename]; -} - -- (NSString *)databaseFilePath_SHM -{ - return [self.databaseDirPath stringByAppendingPathComponent:self.databaseFilename_SHM]; -} - -- (NSString *)databaseFilePath_WAL -{ - return [self.databaseDirPath stringByAppendingPathComponent:self.databaseFilename_WAL]; -} - -- (NSData *)databaseKeySpec -{ - OWSAssert(self.keySpecBlock); - - return self.keySpecBlock(); -} - -- (void)ensureDatabaseKeySpecExists -{ - // Do nothing. -} - -@end - -NS_ASSUME_NONNULL_END From 2c680cadadafd946c78048353090214b0874b9e1 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Sat, 17 Mar 2018 09:37:42 -0300 Subject: [PATCH 07/13] Clean up ahead of PR. --- Signal/src/util/OWSBackupExportJob.m | 36 ++++++++++++++-------------- Signal/src/util/OWSBackupImportJob.m | 12 +++++----- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index ce2bcecded..bfef72fbe0 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -294,7 +294,7 @@ NS_ASSUME_NONNULL_BEGIN progress:nil]; __weak OWSBackupExportJob *weakSelf = self; - [self configureExport:^(BOOL configureExportSuccess) { + [self configureExportWithCompletion:^(BOOL configureExportSuccess) { if (!configureExportSuccess) { [self failWithErrorDescription: NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", @@ -317,12 +317,12 @@ NS_ASSUME_NONNULL_BEGIN if (self.isComplete) { return; } - [self saveToCloud:^(NSError *_Nullable saveError) { + [self saveToCloudWithCompletion:^(NSError *_Nullable saveError) { if (saveError) { [weakSelf failWithError:saveError]; return; } - [self cleanUpCloud:^(NSError *_Nullable cleanUpError) { + [self cleanUpCloudWithCompletion:^(NSError *_Nullable cleanUpError) { if (cleanUpError) { [weakSelf failWithError:cleanUpError]; return; @@ -333,7 +333,7 @@ NS_ASSUME_NONNULL_BEGIN }]; } -- (void)configureExport:(OWSBackupJobBoolCompletion)completion +- (void)configureExportWithCompletion:(OWSBackupJobBoolCompletion)completion { OWSAssert(completion); @@ -526,7 +526,7 @@ NS_ASSUME_NONNULL_BEGIN return YES; } -- (void)saveToCloud:(OWSBackupJobCompletion)completion +- (void)saveToCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); @@ -558,13 +558,13 @@ NS_ASSUME_NONNULL_BEGIN totalFileSize); } - [self saveNextFileToCloud:completion]; + [self saveNextFileToCloudWithCompletion:completion]; } // This method uploads one file (the "next" file) each time it // is called. Each successful file upload re-invokes this method // until the last (the manifest file). -- (void)saveNextFileToCloud:(OWSBackupJobCompletion)completion +- (void)saveNextFileToCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); @@ -581,17 +581,17 @@ NS_ASSUME_NONNULL_BEGIN @"Indicates that the backup export data is being uploaded.") progress:@(progress)]; - if ([self saveNextDatabaseFileToCloud:completion]) { + if ([self saveNextDatabaseFileToCloudWithCompletion:completion]) { return; } - if ([self saveNextAttachmentFileToCloud:completion]) { + if ([self saveNextAttachmentFileToCloudWithCompletion:completion]) { return; } - [self saveManifestFileToCloud:completion]; + [self saveManifestFileToCloudWithCompletion:completion]; } // This method returns YES IFF "work was done and there might be more work to do". -- (BOOL)saveNextDatabaseFileToCloud:(OWSBackupJobCompletion)completion +- (BOOL)saveNextDatabaseFileToCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); @@ -612,7 +612,7 @@ NS_ASSUME_NONNULL_BEGIN dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ item.recordName = recordName; [weakSelf.savedDatabaseItems addObject:item]; - [weakSelf saveNextFileToCloud:completion]; + [weakSelf saveNextFileToCloudWithCompletion:completion]; }); } failure:^(NSError *error) { @@ -624,7 +624,7 @@ NS_ASSUME_NONNULL_BEGIN } // This method returns YES IFF "work was done and there might be more work to do". -- (BOOL)saveNextAttachmentFileToCloud:(OWSBackupJobCompletion)completion +- (BOOL)saveNextAttachmentFileToCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); @@ -642,7 +642,7 @@ NS_ASSUME_NONNULL_BEGIN // attachment to disk. if (![attachmentExport prepareForUpload]) { // Attachment files are non-critical so any error uploading them is recoverable. - [weakSelf saveNextFileToCloud:completion]; + [weakSelf saveNextFileToCloudWithCompletion:completion]; return YES; } OWSAssert(attachmentExport.relativeFilePath.length > 0); @@ -679,21 +679,21 @@ NS_ASSUME_NONNULL_BEGIN self.logTag, attachmentExport.attachmentFilePath, attachmentExport.relativeFilePath); - [strongSelf saveNextFileToCloud:completion]; + [strongSelf saveNextFileToCloudWithCompletion:completion]; }); } failure:^(NSError *error) { // Ensure that we continue to work off the main thread. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ // Attachment files are non-critical so any error uploading them is recoverable. - [weakSelf saveNextFileToCloud:completion]; + [weakSelf saveNextFileToCloudWithCompletion:completion]; }); }]; return YES; } -- (void)saveManifestFileToCloud:(OWSBackupJobCompletion)completion +- (void)saveManifestFileToCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); @@ -780,7 +780,7 @@ NS_ASSUME_NONNULL_BEGIN return result; } -- (void)cleanUpCloud:(OWSBackupJobCompletion)completion +- (void)cleanUpCloudWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); diff --git a/Signal/src/util/OWSBackupImportJob.m b/Signal/src/util/OWSBackupImportJob.m index 78b2859681..99e7cda03f 100644 --- a/Signal/src/util/OWSBackupImportJob.m +++ b/Signal/src/util/OWSBackupImportJob.m @@ -98,7 +98,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe progress:nil]; __weak OWSBackupImportJob *weakSelf = self; - [weakSelf downloadAndProcessManifest:^(NSError *_Nullable manifestError) { + [weakSelf downloadAndProcessManifestWithCompletion:^(NSError *_Nullable manifestError) { if (manifestError) { [weakSelf failWithError:manifestError]; return; @@ -131,7 +131,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return; } - [weakSelf restoreDatabase:^(BOOL restoreDatabaseSuccess) { + [weakSelf restoreDatabaseWithCompletion:^(BOOL restoreDatabaseSuccess) { if (!restoreDatabaseSuccess) { [weakSelf failWithErrorDescription:NSLocalizedString( @"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT", @@ -144,7 +144,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return; } - [weakSelf ensureMigrations:^(BOOL ensureMigrationsSuccess) { + [weakSelf ensureMigrationsWithCompletion:^(BOOL ensureMigrationsSuccess) { if (!ensureMigrationsSuccess) { [weakSelf failWithErrorDescription:NSLocalizedString( @"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT", @@ -178,7 +178,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return YES; } -- (void)downloadAndProcessManifest:(OWSBackupJobCompletion)completion +- (void)downloadAndProcessManifestWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); @@ -422,7 +422,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe } } -- (void)restoreDatabase:(OWSBackupJobBoolCompletion)completion +- (void)restoreDatabaseWithCompletion:(OWSBackupJobBoolCompletion)completion { OWSAssert(completion); @@ -573,7 +573,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe completion(YES); } -- (void)ensureMigrations:(OWSBackupJobBoolCompletion)completion +- (void)ensureMigrationsWithCompletion:(OWSBackupJobBoolCompletion)completion { OWSAssert(completion); From 18d39f15f2f461fe1e1cae26b50eb2fc74e0b371 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Sat, 17 Mar 2018 09:40:50 -0300 Subject: [PATCH 08/13] Clean up ahead of PR. --- Signal/src/network/GiphyAPI.swift | 79 +++++++++++++++---------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/Signal/src/network/GiphyAPI.swift b/Signal/src/network/GiphyAPI.swift index 6bc738dfdf..2ee16daa48 100644 --- a/Signal/src/network/GiphyAPI.swift +++ b/Signal/src/network/GiphyAPI.swift @@ -1,5 +1,5 @@ // -// Copyright (c) 2018 Open Whisper Systems. All rights reserved. +// Copyright (c) 2017 Open Whisper Systems. All rights reserved. // import Foundation @@ -122,33 +122,33 @@ extension GiphyError: LocalizedError { public func pickStillRendition() -> GiphyRendition? { // Stills are just temporary placeholders, so use the smallest still possible. - return pickRendition(renditionType: .stillPreview, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize) + return pickRendition(renditionType: .stillPreview, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize) } public func pickPreviewRendition() -> GiphyRendition? { // Try to pick a small file... - if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .largerIsBetter, maxFileSize: kPreferedPreviewFileSize) { + if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.largerIsBetter, maxFileSize:kPreferedPreviewFileSize) { return rendition } // ...but gradually relax the file restriction... - if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize * 2) { + if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize * 2) { return rendition } // ...and relax even more until we find an animated rendition. - return pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize * 3) + return pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize * 3) } public func pickSendingRendition() -> GiphyRendition? { // Try to pick a small file... - if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .largerIsBetter, maxFileSize: kPreferedSendingFileSize) { + if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.largerIsBetter, maxFileSize:kPreferedSendingFileSize) { return rendition } // ...but gradually relax the file restriction... - if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedSendingFileSize * 2) { + if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedSendingFileSize * 2) { return rendition } // ...and relax even more until we find an animated rendition. - return pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedSendingFileSize * 3) + return pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedSendingFileSize * 3) } enum RenditionType { @@ -299,12 +299,12 @@ extension GiphyError: LocalizedError { } private func giphyAPISessionManager() -> AFHTTPSessionManager? { - guard let baseUrl = NSURL(string: kGiphyBaseURL) else { + guard let baseUrl = NSURL(string:kGiphyBaseURL) else { Logger.error("\(TAG) Invalid base URL.") return nil } - let sessionManager = AFHTTPSessionManager(baseURL: baseUrl as URL, - sessionConfiguration: GiphyAPI.giphySessionConfiguration()) + let sessionManager = AFHTTPSessionManager(baseURL:baseUrl as URL, + sessionConfiguration:GiphyAPI.giphySessionConfiguration()) sessionManager.requestSerializer = AFJSONRequestSerializer() sessionManager.responseSerializer = AFJSONResponseSerializer() @@ -319,7 +319,7 @@ extension GiphyError: LocalizedError { failure(nil) return } - guard NSURL(string: kGiphyBaseURL) != nil else { + guard NSURL(string:kGiphyBaseURL) != nil else { Logger.error("\(TAG) Invalid base URL.") failure(nil) return @@ -338,11 +338,10 @@ extension GiphyError: LocalizedError { sessionManager.get(urlString, parameters: {}, - progress: nil, - success: { task, value in + progress:nil, + success: { _, value in Logger.error("\(GiphyAPI.TAG) search request succeeded") - Logger.error("\(GiphyAPI.TAG) search request succeeded \(task.originalRequest?.url)") - guard let imageInfos = self.parseGiphyImages(responseJson: value) else { + guard let imageInfos = self.parseGiphyImages(responseJson:value) else { failure(nil) return } @@ -356,16 +355,16 @@ extension GiphyError: LocalizedError { // MARK: Parse API Responses - private func parseGiphyImages(responseJson: Any?) -> [GiphyImageInfo]? { + private func parseGiphyImages(responseJson:Any?) -> [GiphyImageInfo]? { guard let responseJson = responseJson else { Logger.error("\(TAG) Missing response.") return nil } - guard let responseDict = responseJson as? [String: Any] else { + guard let responseDict = responseJson as? [String:Any] else { Logger.error("\(TAG) Invalid response.") return nil } - guard let imageDicts = responseDict["data"] as? [[String: Any]] else { + guard let imageDicts = responseDict["data"] as? [[String:Any]] else { Logger.error("\(TAG) Invalid response data.") return nil } @@ -375,7 +374,7 @@ extension GiphyError: LocalizedError { } // Giphy API results are often incomplete or malformed, so we need to be defensive. - private func parseGiphyImage(imageDict: [String: Any]) -> GiphyImageInfo? { + private func parseGiphyImage(imageDict: [String:Any]) -> GiphyImageInfo? { guard let giphyId = imageDict["id"] as? String else { Logger.warn("\(TAG) Image dict missing id.") return nil @@ -384,18 +383,18 @@ extension GiphyError: LocalizedError { Logger.warn("\(TAG) Image dict has invalid id.") return nil } - guard let renditionDicts = imageDict["images"] as? [String: Any] else { + guard let renditionDicts = imageDict["images"] as? [String:Any] else { Logger.warn("\(TAG) Image dict missing renditions.") return nil } var renditions = [GiphyRendition]() for (renditionName, renditionDict) in renditionDicts { - guard let renditionDict = renditionDict as? [String: Any] else { + guard let renditionDict = renditionDict as? [String:Any] else { Logger.warn("\(TAG) Invalid rendition dict.") continue } - guard let rendition = parseGiphyRendition(renditionName: renditionName, - renditionDict: renditionDict) else { + guard let rendition = parseGiphyRendition(renditionName:renditionName, + renditionDict:renditionDict) else { continue } renditions.append(rendition) @@ -405,13 +404,13 @@ extension GiphyError: LocalizedError { return nil } - guard let originalRendition = findOriginalRendition(renditions: renditions) else { + guard let originalRendition = findOriginalRendition(renditions:renditions) else { Logger.warn("\(TAG) Image has no original rendition.") return nil } - return GiphyImageInfo(giphyId: giphyId, - renditions: renditions, + return GiphyImageInfo(giphyId : giphyId, + renditions : renditions, originalRendition: originalRendition) } @@ -426,15 +425,15 @@ extension GiphyError: LocalizedError { // // We should discard renditions which are missing or have invalid properties. private func parseGiphyRendition(renditionName: String, - renditionDict: [String: Any]) -> GiphyRendition? { - guard let width = parsePositiveUInt(dict: renditionDict, key: "width", typeName: "rendition") else { + renditionDict: [String:Any]) -> GiphyRendition? { + guard let width = parsePositiveUInt(dict:renditionDict, key:"width", typeName:"rendition") else { return nil } - guard let height = parsePositiveUInt(dict: renditionDict, key: "height", typeName: "rendition") else { + guard let height = parsePositiveUInt(dict:renditionDict, key:"height", typeName:"rendition") else { return nil } // Be lenient when parsing file sizes - we don't require them for stills. - let fileSize = parseLenientUInt(dict: renditionDict, key: "size") + let fileSize = parseLenientUInt(dict:renditionDict, key:"size") guard let urlString = renditionDict["url"] as? String else { return nil } @@ -442,7 +441,7 @@ extension GiphyError: LocalizedError { Logger.warn("\(TAG) Rendition has invalid url.") return nil } - guard let url = NSURL(string: urlString) else { + guard let url = NSURL(string:urlString) else { Logger.warn("\(TAG) Rendition url could not be parsed.") return nil } @@ -465,16 +464,16 @@ extension GiphyError: LocalizedError { } return GiphyRendition( - format: format, - name: renditionName, - width: width, - height: height, - fileSize: fileSize, - url: url + format : format, + name : renditionName, + width : width, + height : height, + fileSize : fileSize, + url : url ) } - private func parsePositiveUInt(dict: [String: Any], key: String, typeName: String) -> UInt? { + private func parsePositiveUInt(dict: [String:Any], key: String, typeName: String) -> UInt? { guard let value = dict[key] else { return nil } @@ -491,7 +490,7 @@ extension GiphyError: LocalizedError { return parsedValue } - private func parseLenientUInt(dict: [String: Any], key: String) -> UInt { + private func parseLenientUInt(dict: [String:Any], key: String) -> UInt { let defaultValue = UInt(0) guard let value = dict[key] else { From 08ba7c85ed11dc48da187f201c07152ad8c3667a Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Sat, 17 Mar 2018 10:03:52 -0300 Subject: [PATCH 09/13] Clean up ahead of PR. --- Signal/src/util/OWSBackupExportJob.m | 53 ++++++++++++++++++++--- Signal/src/util/OWSBackupIO.h | 5 +++ SignalServiceKit/src/Util/OWSFileSystem.h | 4 +- SignalServiceKit/src/Util/OWSFileSystem.m | 15 ++++--- 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index bfef72fbe0..44431e486c 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -28,14 +28,12 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic) NSString *recordName; -// This property is optional and represents the location of this -// item relative to the root directory for items of this type. -//@property (nonatomic, nullable) NSString *fileRelativePath; - // This property is optional and is only used for attachments. @property (nonatomic, nullable) OWSAttachmentExport *attachmentExport; // This property is optional. +// +// See comments in `OWSBackupIO`. @property (nonatomic, nullable) NSNumber *uncompressedDataLength; - (instancetype)init NS_UNAVAILABLE; @@ -63,6 +61,19 @@ NS_ASSUME_NONNULL_BEGIN #pragma mark - +// Used to serialize database snapshot contents. +// Writes db entities using protobufs into snapshot fragments. +// Snapshot fragments are compressed (they compress _very well_, +// around 20x smaller) then encrypted. Ordering matters in +// snapshot contents (entities should we restored in the same +// order they are serialized), so we are always careful to preserve +// ordering of entities within a snapshot AND ordering of snapshot +// fragments within a bakckup. +// +// This stream is used to write entities one at a time and takes +// care of sharding them into fragments, compressing and encrypting +// those fragments. Fragment size is fixed to reduce worst case +// memory usage. @interface OWSDBExportStream : NSObject @property (nonatomic) OWSBackupIO *backupIO; @@ -97,6 +108,10 @@ NS_ASSUME_NONNULL_BEGIN return self; } + +// It isn't strictly necessary to capture the entity type (the importer doesn't +// use this state), but I think it'll be helpful to have around to future-proof +// this work, help with debugging issue, etc. - (BOOL)writeObject:(TSYapDatabaseObject *)object entityType:(OWSSignalServiceProtosBackupSnapshotBackupEntityType)entityType { @@ -133,15 +148,17 @@ NS_ASSUME_NONNULL_BEGIN } // Write cached data to disk, if necessary. +// +// Returns YES on success. - (BOOL)flush { if (!self.backupSnapshotBuilder) { + // No data to flush to disk. return YES; } // Try to release allocated buffers ASAP. @autoreleasepool { - NSData *_Nullable uncompressedData = [self.backupSnapshotBuilder build].data; NSUInteger uncompressedDataLength = uncompressedData.length; self.backupSnapshotBuilder = nil; @@ -171,6 +188,12 @@ NS_ASSUME_NONNULL_BEGIN #pragma mark - +// This class is used to: +// +// * Lazy-encrypt and eagerly cleanup attachment uploads. +// To reduce disk footprint of backup export process, +// we only want to have one attachment export on disk +// at a time. @interface OWSAttachmentExport : NSObject @property (nonatomic) OWSBackupIO *backupIO; @@ -210,9 +233,13 @@ NS_ASSUME_NONNULL_BEGIN { // Surface memory leaks by logging the deallocation. DDLogVerbose(@"Dealloc: %@", self.class); + + [self cleanUp]; } // On success, encryptedItem will be non-nil. +// +// Returns YES on success. - (BOOL)prepareForUpload { OWSAssert(self.attachmentId.length > 0); @@ -241,6 +268,12 @@ NS_ASSUME_NONNULL_BEGIN return YES; } +// Returns YES on success. +- (BOOL)cleanUp +{ + return [OWSFileSystem deleteFileIfExists:self.encryptedItem.filePath]; +} + @end #pragma mark - @@ -669,6 +702,11 @@ NS_ASSUME_NONNULL_BEGIN return; } + if (![attachmentExport cleanUp]) { + DDLogError(@"%@ couldn't clean up attachment export.", self.logTag); + // Attachment files are non-critical so any error uploading them is recoverable. + } + OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; exportItem.encryptedItem = attachmentExport.encryptedItem; exportItem.recordName = recordName; @@ -685,6 +723,11 @@ NS_ASSUME_NONNULL_BEGIN failure:^(NSError *error) { // Ensure that we continue to work off the main thread. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + if (![attachmentExport cleanUp]) { + DDLogError(@"%@ couldn't clean up attachment export.", self.logTag); + // Attachment files are non-critical so any error uploading them is recoverable. + } + // Attachment files are non-critical so any error uploading them is recoverable. [weakSelf saveNextFileToCloudWithCompletion:completion]; }); diff --git a/Signal/src/util/OWSBackupIO.h b/Signal/src/util/OWSBackupIO.h index 38414fa06f..fabb0088d3 100644 --- a/Signal/src/util/OWSBackupIO.h +++ b/Signal/src/util/OWSBackupIO.h @@ -45,6 +45,11 @@ NS_ASSUME_NONNULL_BEGIN - (nullable NSData *)compressData:(NSData *)srcData; +// I'm using the (new in iOS 9) compressionlib. One of its weaknesses is that it +// requires you to pre-allocate output buffers during compression and decompression. +// During decompression this is particularly tricky since there's no way to safely +// predict how large the output will be based on the input. So, we store the +// uncompressed size for compressed backup items. - (nullable NSData *)decompressData:(NSData *)srcData uncompressedDataLength:(NSUInteger)uncompressedDataLength; @end diff --git a/SignalServiceKit/src/Util/OWSFileSystem.h b/SignalServiceKit/src/Util/OWSFileSystem.h index 70ec5492bf..e9a1802898 100644 --- a/SignalServiceKit/src/Util/OWSFileSystem.h +++ b/SignalServiceKit/src/Util/OWSFileSystem.h @@ -28,9 +28,9 @@ NS_ASSUME_NONNULL_BEGIN // Returns NO IFF the directory does not exist and could not be created. + (BOOL)ensureDirectoryExists:(NSString *)dirPath; -+ (void)deleteFile:(NSString *)filePath; ++ (BOOL)deleteFile:(NSString *)filePath; -+ (void)deleteFileIfExists:(NSString *)filePath; ++ (BOOL)deleteFileIfExists:(NSString *)filePath; + (NSArray *_Nullable)allFilesInDirectoryRecursive:(NSString *)dirPath error:(NSError **)error; diff --git a/SignalServiceKit/src/Util/OWSFileSystem.m b/SignalServiceKit/src/Util/OWSFileSystem.m index ddcb724e9d..fa83b41a39 100644 --- a/SignalServiceKit/src/Util/OWSFileSystem.m +++ b/SignalServiceKit/src/Util/OWSFileSystem.m @@ -227,20 +227,23 @@ NS_ASSUME_NONNULL_BEGIN } } -+ (void)deleteFile:(NSString *)filePath ++ (BOOL)deleteFile:(NSString *)filePath { NSError *error; - [[NSFileManager defaultManager] removeItemAtPath:filePath error:&error]; - if (error) { + BOOL success = [[NSFileManager defaultManager] removeItemAtPath:filePath error:&error]; + if (!success || error) { DDLogError(@"%@ Failed to delete file: %@", self.logTag, error.description); + return NO; } + return YES; } -+ (void)deleteFileIfExists:(NSString *)filePath ++ (BOOL)deleteFileIfExists:(NSString *)filePath { - if ([[NSFileManager defaultManager] fileExistsAtPath:filePath]) { - [self deleteFile:filePath]; + if (![[NSFileManager defaultManager] fileExistsAtPath:filePath]) { + return YES; } + return [self deleteFile:filePath]; } + (NSArray *_Nullable)allFilesInDirectoryRecursive:(NSString *)dirPath error:(NSError **)error From ab720a310088d6ee89ebfb9683ac33ebe873669e Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Tue, 20 Mar 2018 11:06:27 -0400 Subject: [PATCH 10/13] Move backup protos to their own proto schema. --- .../protobuf/OWSSignalServiceProtos.proto | 18 - .../protobuf/OWSSignaliOSProtos.proto | 29 + .../src/Messages/OWSSignalServiceProtos.pb.h | 125 ---- .../src/Messages/OWSSignalServiceProtos.pb.m | 510 ----------------- .../src/Protos/OWSSignaliOSProtos.pb.h | 186 ++++++ .../src/Protos/OWSSignaliOSProtos.pb.m | 535 ++++++++++++++++++ 6 files changed, 750 insertions(+), 653 deletions(-) create mode 100644 SignalServiceKit/protobuf/OWSSignaliOSProtos.proto create mode 100644 SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.h create mode 100644 SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.m diff --git a/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto b/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto index c2e01fe34e..263ff798c2 100644 --- a/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto +++ b/SignalServiceKit/protobuf/OWSSignalServiceProtos.proto @@ -238,21 +238,3 @@ message GroupDetails { optional bool active = 5 [default = true]; optional uint32 expireTimer = 6; } - -message BackupSnapshot -{ - message BackupEntity - { - enum Type { - UNKNOWN = 0; - MIGRATION = 1; - THREAD = 2; - INTERACTION = 3; - ATTACHMENT = 4; - } - optional Type type = 1; - optional bytes entityData = 2; - } - - repeated BackupEntity entity = 1; -} \ No newline at end of file diff --git a/SignalServiceKit/protobuf/OWSSignaliOSProtos.proto b/SignalServiceKit/protobuf/OWSSignaliOSProtos.proto new file mode 100644 index 0000000000..901b598dc9 --- /dev/null +++ b/SignalServiceKit/protobuf/OWSSignaliOSProtos.proto @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2014-2016 Open Whisper Systems + * + * Licensed according to the LICENSE file in this repository. + */ + +package signalios; + +// Signal-iOS +import "objectivec-descriptor.proto"; +option (google.protobuf.objectivec_file_options).class_prefix = "OWSSignaliOSProtos"; + +message BackupSnapshot +{ + message BackupEntity + { + enum Type { + UNKNOWN = 0; + MIGRATION = 1; + THREAD = 2; + INTERACTION = 3; + ATTACHMENT = 4; + } + optional Type type = 1; + optional bytes entityData = 2; + } + + repeated BackupEntity entity = 1; +} \ No newline at end of file diff --git a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h index 494e171aec..2307512a3a 100644 --- a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h +++ b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.h @@ -6,10 +6,6 @@ @class OWSSignalServiceProtosAttachmentPointer; @class OWSSignalServiceProtosAttachmentPointerBuilder; -@class OWSSignalServiceProtosBackupSnapshot; -@class OWSSignalServiceProtosBackupSnapshotBackupEntity; -@class OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder; -@class OWSSignalServiceProtosBackupSnapshotBuilder; @class OWSSignalServiceProtosCallMessage; @class OWSSignalServiceProtosCallMessageAnswer; @class OWSSignalServiceProtosCallMessageAnswerBuilder; @@ -172,17 +168,6 @@ typedef NS_ENUM(SInt32, OWSSignalServiceProtosGroupContextType) { BOOL OWSSignalServiceProtosGroupContextTypeIsValidValue(OWSSignalServiceProtosGroupContextType value); NSString *NSStringFromOWSSignalServiceProtosGroupContextType(OWSSignalServiceProtosGroupContextType value); -typedef NS_ENUM(SInt32, OWSSignalServiceProtosBackupSnapshotBackupEntityType) { - OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown = 0, - OWSSignalServiceProtosBackupSnapshotBackupEntityTypeMigration = 1, - OWSSignalServiceProtosBackupSnapshotBackupEntityTypeThread = 2, - OWSSignalServiceProtosBackupSnapshotBackupEntityTypeInteraction = 3, - OWSSignalServiceProtosBackupSnapshotBackupEntityTypeAttachment = 4, -}; - -BOOL OWSSignalServiceProtosBackupSnapshotBackupEntityTypeIsValidValue(OWSSignalServiceProtosBackupSnapshotBackupEntityType value); -NSString *NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(OWSSignalServiceProtosBackupSnapshotBackupEntityType value); - @interface OWSSignalServiceProtosOwssignalServiceProtosRoot : NSObject { } @@ -2239,115 +2224,5 @@ NSString *NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(OWSSi - (OWSSignalServiceProtosGroupDetailsBuilder*) clearExpireTimer; @end -#define BackupSnapshot_entity @"entity" -@interface OWSSignalServiceProtosBackupSnapshot : PBGeneratedMessage { -@private - NSMutableArray * entityArray; -} -@property (readonly, strong) NSArray * entity; -- (OWSSignalServiceProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index; - -+ (instancetype) defaultInstance; -- (instancetype) defaultInstance; - -- (BOOL) isInitialized; -- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output; -- (OWSSignalServiceProtosBackupSnapshotBuilder*) builder; -+ (OWSSignalServiceProtosBackupSnapshotBuilder*) builder; -+ (OWSSignalServiceProtosBackupSnapshotBuilder*) builderWithPrototype:(OWSSignalServiceProtosBackupSnapshot*) prototype; -- (OWSSignalServiceProtosBackupSnapshotBuilder*) toBuilder; - -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromData:(NSData*) data; -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry; -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input; -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input; -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; -@end - -#define BackupEntity_type @"type" -#define BackupEntity_entityData @"entityData" -@interface OWSSignalServiceProtosBackupSnapshotBackupEntity : PBGeneratedMessage { -@private - BOOL hasEntityData_:1; - BOOL hasType_:1; - NSData* entityData; - OWSSignalServiceProtosBackupSnapshotBackupEntityType type; -} -- (BOOL) hasType; -- (BOOL) hasEntityData; -@property (readonly) OWSSignalServiceProtosBackupSnapshotBackupEntityType type; -@property (readonly, strong) NSData* entityData; - -+ (instancetype) defaultInstance; -- (instancetype) defaultInstance; - -- (BOOL) isInitialized; -- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builder; -+ (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builder; -+ (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builderWithPrototype:(OWSSignalServiceProtosBackupSnapshotBackupEntity*) prototype; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) toBuilder; - -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data; -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry; -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input; -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input; -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; -@end - -@interface OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder : PBGeneratedMessageBuilder { -@private - OWSSignalServiceProtosBackupSnapshotBackupEntity* resultBackupEntity; -} - -- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) defaultInstance; - -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clear; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clone; - -- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) build; -- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) buildPartial; - -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFrom:(OWSSignalServiceProtosBackupSnapshotBackupEntity*) other; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; - -- (BOOL) hasType; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityType) type; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setType:(OWSSignalServiceProtosBackupSnapshotBackupEntityType) value; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearType; - -- (BOOL) hasEntityData; -- (NSData*) entityData; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setEntityData:(NSData*) value; -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearEntityData; -@end - -@interface OWSSignalServiceProtosBackupSnapshotBuilder : PBGeneratedMessageBuilder { -@private - OWSSignalServiceProtosBackupSnapshot* resultBackupSnapshot; -} - -- (OWSSignalServiceProtosBackupSnapshot*) defaultInstance; - -- (OWSSignalServiceProtosBackupSnapshotBuilder*) clear; -- (OWSSignalServiceProtosBackupSnapshotBuilder*) clone; - -- (OWSSignalServiceProtosBackupSnapshot*) build; -- (OWSSignalServiceProtosBackupSnapshot*) buildPartial; - -- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFrom:(OWSSignalServiceProtosBackupSnapshot*) other; -- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input; -- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; - -- (NSMutableArray *)entity; -- (OWSSignalServiceProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index; -- (OWSSignalServiceProtosBackupSnapshotBuilder *)addEntity:(OWSSignalServiceProtosBackupSnapshotBackupEntity*)value; -- (OWSSignalServiceProtosBackupSnapshotBuilder *)setEntityArray:(NSArray *)array; -- (OWSSignalServiceProtosBackupSnapshotBuilder *)clearEntity; -@end - // @@protoc_insertion_point(global_scope) diff --git a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m index 1770d2abe6..9b81f29081 100644 --- a/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m +++ b/SignalServiceKit/src/Messages/OWSSignalServiceProtos.pb.m @@ -9705,515 +9705,5 @@ static OWSSignalServiceProtosGroupDetailsAvatar* defaultOWSSignalServiceProtosGr } @end -@interface OWSSignalServiceProtosBackupSnapshot () -@property (strong) NSMutableArray * entityArray; -@end - -@implementation OWSSignalServiceProtosBackupSnapshot - -@synthesize entityArray; -@dynamic entity; -- (instancetype) init { - if ((self = [super init])) { - } - return self; -} -static OWSSignalServiceProtosBackupSnapshot* defaultOWSSignalServiceProtosBackupSnapshotInstance = nil; -+ (void) initialize { - if (self == [OWSSignalServiceProtosBackupSnapshot class]) { - defaultOWSSignalServiceProtosBackupSnapshotInstance = [[OWSSignalServiceProtosBackupSnapshot alloc] init]; - } -} -+ (instancetype) defaultInstance { - return defaultOWSSignalServiceProtosBackupSnapshotInstance; -} -- (instancetype) defaultInstance { - return defaultOWSSignalServiceProtosBackupSnapshotInstance; -} -- (NSArray *)entity { - return entityArray; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index { - return [entityArray objectAtIndex:index]; -} -- (BOOL) isInitialized { - return YES; -} -- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output { - [self.entityArray enumerateObjectsUsingBlock:^(OWSSignalServiceProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { - [output writeMessage:1 value:element]; - }]; - [self.unknownFields writeToCodedOutputStream:output]; -} -- (SInt32) serializedSize { - __block SInt32 size_ = memoizedSerializedSize; - if (size_ != -1) { - return size_; - } - - size_ = 0; - [self.entityArray enumerateObjectsUsingBlock:^(OWSSignalServiceProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { - size_ += computeMessageSize(1, element); - }]; - size_ += self.unknownFields.serializedSize; - memoizedSerializedSize = size_; - return size_; -} -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromData:(NSData*) data { - return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromData:data] build]; -} -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry { - return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromData:data extensionRegistry:extensionRegistry] build]; -} -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input { - return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromInputStream:input] build]; -} -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { - return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromInputStream:input extensionRegistry:extensionRegistry] build]; -} -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input { - return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromCodedInputStream:input] build]; -} -+ (OWSSignalServiceProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { - return (OWSSignalServiceProtosBackupSnapshot*)[[[OWSSignalServiceProtosBackupSnapshot builder] mergeFromCodedInputStream:input extensionRegistry:extensionRegistry] build]; -} -+ (OWSSignalServiceProtosBackupSnapshotBuilder*) builder { - return [[OWSSignalServiceProtosBackupSnapshotBuilder alloc] init]; -} -+ (OWSSignalServiceProtosBackupSnapshotBuilder*) builderWithPrototype:(OWSSignalServiceProtosBackupSnapshot*) prototype { - return [[OWSSignalServiceProtosBackupSnapshot builder] mergeFrom:prototype]; -} -- (OWSSignalServiceProtosBackupSnapshotBuilder*) builder { - return [OWSSignalServiceProtosBackupSnapshot builder]; -} -- (OWSSignalServiceProtosBackupSnapshotBuilder*) toBuilder { - return [OWSSignalServiceProtosBackupSnapshot builderWithPrototype:self]; -} -- (void) writeDescriptionTo:(NSMutableString*) output withIndent:(NSString*) indent { - [self.entityArray enumerateObjectsUsingBlock:^(OWSSignalServiceProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { - [output appendFormat:@"%@%@ {\n", indent, @"entity"]; - [element writeDescriptionTo:output - withIndent:[NSString stringWithFormat:@"%@ ", indent]]; - [output appendFormat:@"%@}\n", indent]; - }]; - [self.unknownFields writeDescriptionTo:output withIndent:indent]; -} -- (void) storeInDictionary:(NSMutableDictionary *)dictionary { - for (OWSSignalServiceProtosBackupSnapshotBackupEntity* element in self.entityArray) { - NSMutableDictionary *elementDictionary = [NSMutableDictionary dictionary]; - [element storeInDictionary:elementDictionary]; - [dictionary setObject:[NSDictionary dictionaryWithDictionary:elementDictionary] forKey:@"entity"]; - } - [self.unknownFields storeInDictionary:dictionary]; -} -- (BOOL) isEqual:(id)other { - if (other == self) { - return YES; - } - if (![other isKindOfClass:[OWSSignalServiceProtosBackupSnapshot class]]) { - return NO; - } - OWSSignalServiceProtosBackupSnapshot *otherMessage = other; - return - [self.entityArray isEqualToArray:otherMessage.entityArray] && - (self.unknownFields == otherMessage.unknownFields || (self.unknownFields != nil && [self.unknownFields isEqual:otherMessage.unknownFields])); -} -- (NSUInteger) hash { - __block NSUInteger hashCode = 7; - [self.entityArray enumerateObjectsUsingBlock:^(OWSSignalServiceProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { - hashCode = hashCode * 31 + [element hash]; - }]; - hashCode = hashCode * 31 + [self.unknownFields hash]; - return hashCode; -} -@end - -@interface OWSSignalServiceProtosBackupSnapshotBackupEntity () -@property OWSSignalServiceProtosBackupSnapshotBackupEntityType type; -@property (strong) NSData* entityData; -@end - -@implementation OWSSignalServiceProtosBackupSnapshotBackupEntity - -- (BOOL) hasType { - return !!hasType_; -} -- (void) setHasType:(BOOL) _value_ { - hasType_ = !!_value_; -} -@synthesize type; -- (BOOL) hasEntityData { - return !!hasEntityData_; -} -- (void) setHasEntityData:(BOOL) _value_ { - hasEntityData_ = !!_value_; -} -@synthesize entityData; -- (instancetype) init { - if ((self = [super init])) { - self.type = OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown; - self.entityData = [NSData data]; - } - return self; -} -static OWSSignalServiceProtosBackupSnapshotBackupEntity* defaultOWSSignalServiceProtosBackupSnapshotBackupEntityInstance = nil; -+ (void) initialize { - if (self == [OWSSignalServiceProtosBackupSnapshotBackupEntity class]) { - defaultOWSSignalServiceProtosBackupSnapshotBackupEntityInstance = [[OWSSignalServiceProtosBackupSnapshotBackupEntity alloc] init]; - } -} -+ (instancetype) defaultInstance { - return defaultOWSSignalServiceProtosBackupSnapshotBackupEntityInstance; -} -- (instancetype) defaultInstance { - return defaultOWSSignalServiceProtosBackupSnapshotBackupEntityInstance; -} -- (BOOL) isInitialized { - return YES; -} -- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output { - if (self.hasType) { - [output writeEnum:1 value:self.type]; - } - if (self.hasEntityData) { - [output writeData:2 value:self.entityData]; - } - [self.unknownFields writeToCodedOutputStream:output]; -} -- (SInt32) serializedSize { - __block SInt32 size_ = memoizedSerializedSize; - if (size_ != -1) { - return size_; - } - - size_ = 0; - if (self.hasType) { - size_ += computeEnumSize(1, self.type); - } - if (self.hasEntityData) { - size_ += computeDataSize(2, self.entityData); - } - size_ += self.unknownFields.serializedSize; - memoizedSerializedSize = size_; - return size_; -} -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data { - return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromData:data] build]; -} -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry { - return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromData:data extensionRegistry:extensionRegistry] build]; -} -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input { - return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromInputStream:input] build]; -} -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { - return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromInputStream:input extensionRegistry:extensionRegistry] build]; -} -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input { - return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromCodedInputStream:input] build]; -} -+ (OWSSignalServiceProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { - return (OWSSignalServiceProtosBackupSnapshotBackupEntity*)[[[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFromCodedInputStream:input extensionRegistry:extensionRegistry] build]; -} -+ (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builder { - return [[OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder alloc] init]; -} -+ (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builderWithPrototype:(OWSSignalServiceProtosBackupSnapshotBackupEntity*) prototype { - return [[OWSSignalServiceProtosBackupSnapshotBackupEntity builder] mergeFrom:prototype]; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) builder { - return [OWSSignalServiceProtosBackupSnapshotBackupEntity builder]; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) toBuilder { - return [OWSSignalServiceProtosBackupSnapshotBackupEntity builderWithPrototype:self]; -} -- (void) writeDescriptionTo:(NSMutableString*) output withIndent:(NSString*) indent { - if (self.hasType) { - [output appendFormat:@"%@%@: %@\n", indent, @"type", NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(self.type)]; - } - if (self.hasEntityData) { - [output appendFormat:@"%@%@: %@\n", indent, @"entityData", self.entityData]; - } - [self.unknownFields writeDescriptionTo:output withIndent:indent]; -} -- (void) storeInDictionary:(NSMutableDictionary *)dictionary { - if (self.hasType) { - [dictionary setObject: @(self.type) forKey: @"type"]; - } - if (self.hasEntityData) { - [dictionary setObject: self.entityData forKey: @"entityData"]; - } - [self.unknownFields storeInDictionary:dictionary]; -} -- (BOOL) isEqual:(id)other { - if (other == self) { - return YES; - } - if (![other isKindOfClass:[OWSSignalServiceProtosBackupSnapshotBackupEntity class]]) { - return NO; - } - OWSSignalServiceProtosBackupSnapshotBackupEntity *otherMessage = other; - return - self.hasType == otherMessage.hasType && - (!self.hasType || self.type == otherMessage.type) && - self.hasEntityData == otherMessage.hasEntityData && - (!self.hasEntityData || [self.entityData isEqual:otherMessage.entityData]) && - (self.unknownFields == otherMessage.unknownFields || (self.unknownFields != nil && [self.unknownFields isEqual:otherMessage.unknownFields])); -} -- (NSUInteger) hash { - __block NSUInteger hashCode = 7; - if (self.hasType) { - hashCode = hashCode * 31 + self.type; - } - if (self.hasEntityData) { - hashCode = hashCode * 31 + [self.entityData hash]; - } - hashCode = hashCode * 31 + [self.unknownFields hash]; - return hashCode; -} -@end - -BOOL OWSSignalServiceProtosBackupSnapshotBackupEntityTypeIsValidValue(OWSSignalServiceProtosBackupSnapshotBackupEntityType value) { - switch (value) { - case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown: - case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeMigration: - case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeThread: - case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeInteraction: - case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeAttachment: - return YES; - default: - return NO; - } -} -NSString *NSStringFromOWSSignalServiceProtosBackupSnapshotBackupEntityType(OWSSignalServiceProtosBackupSnapshotBackupEntityType value) { - switch (value) { - case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown: - return @"OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown"; - case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeMigration: - return @"OWSSignalServiceProtosBackupSnapshotBackupEntityTypeMigration"; - case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeThread: - return @"OWSSignalServiceProtosBackupSnapshotBackupEntityTypeThread"; - case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeInteraction: - return @"OWSSignalServiceProtosBackupSnapshotBackupEntityTypeInteraction"; - case OWSSignalServiceProtosBackupSnapshotBackupEntityTypeAttachment: - return @"OWSSignalServiceProtosBackupSnapshotBackupEntityTypeAttachment"; - default: - return nil; - } -} - -@interface OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder() -@property (strong) OWSSignalServiceProtosBackupSnapshotBackupEntity* resultBackupEntity; -@end - -@implementation OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder -@synthesize resultBackupEntity; -- (instancetype) init { - if ((self = [super init])) { - self.resultBackupEntity = [[OWSSignalServiceProtosBackupSnapshotBackupEntity alloc] init]; - } - return self; -} -- (PBGeneratedMessage*) internalGetResult { - return resultBackupEntity; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clear { - self.resultBackupEntity = [[OWSSignalServiceProtosBackupSnapshotBackupEntity alloc] init]; - return self; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clone { - return [OWSSignalServiceProtosBackupSnapshotBackupEntity builderWithPrototype:resultBackupEntity]; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) defaultInstance { - return [OWSSignalServiceProtosBackupSnapshotBackupEntity defaultInstance]; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) build { - [self checkInitialized]; - return [self buildPartial]; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntity*) buildPartial { - OWSSignalServiceProtosBackupSnapshotBackupEntity* returnMe = resultBackupEntity; - self.resultBackupEntity = nil; - return returnMe; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFrom:(OWSSignalServiceProtosBackupSnapshotBackupEntity*) other { - if (other == [OWSSignalServiceProtosBackupSnapshotBackupEntity defaultInstance]) { - return self; - } - if (other.hasType) { - [self setType:other.type]; - } - if (other.hasEntityData) { - [self setEntityData:other.entityData]; - } - [self mergeUnknownFields:other.unknownFields]; - return self; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input { - return [self mergeFromCodedInputStream:input extensionRegistry:[PBExtensionRegistry emptyRegistry]]; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { - PBUnknownFieldSetBuilder* unknownFields = [PBUnknownFieldSet builderWithUnknownFields:self.unknownFields]; - while (YES) { - SInt32 tag = [input readTag]; - switch (tag) { - case 0: - [self setUnknownFields:[unknownFields build]]; - return self; - default: { - if (![self parseUnknownField:input unknownFields:unknownFields extensionRegistry:extensionRegistry tag:tag]) { - [self setUnknownFields:[unknownFields build]]; - return self; - } - break; - } - case 8: { - OWSSignalServiceProtosBackupSnapshotBackupEntityType value = (OWSSignalServiceProtosBackupSnapshotBackupEntityType)[input readEnum]; - if (OWSSignalServiceProtosBackupSnapshotBackupEntityTypeIsValidValue(value)) { - [self setType:value]; - } else { - [unknownFields mergeVarintField:1 value:value]; - } - break; - } - case 18: { - [self setEntityData:[input readData]]; - break; - } - } - } -} -- (BOOL) hasType { - return resultBackupEntity.hasType; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityType) type { - return resultBackupEntity.type; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setType:(OWSSignalServiceProtosBackupSnapshotBackupEntityType) value { - resultBackupEntity.hasType = YES; - resultBackupEntity.type = value; - return self; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearType { - resultBackupEntity.hasType = NO; - resultBackupEntity.type = OWSSignalServiceProtosBackupSnapshotBackupEntityTypeUnknown; - return self; -} -- (BOOL) hasEntityData { - return resultBackupEntity.hasEntityData; -} -- (NSData*) entityData { - return resultBackupEntity.entityData; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) setEntityData:(NSData*) value { - resultBackupEntity.hasEntityData = YES; - resultBackupEntity.entityData = value; - return self; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder*) clearEntityData { - resultBackupEntity.hasEntityData = NO; - resultBackupEntity.entityData = [NSData data]; - return self; -} -@end - -@interface OWSSignalServiceProtosBackupSnapshotBuilder() -@property (strong) OWSSignalServiceProtosBackupSnapshot* resultBackupSnapshot; -@end - -@implementation OWSSignalServiceProtosBackupSnapshotBuilder -@synthesize resultBackupSnapshot; -- (instancetype) init { - if ((self = [super init])) { - self.resultBackupSnapshot = [[OWSSignalServiceProtosBackupSnapshot alloc] init]; - } - return self; -} -- (PBGeneratedMessage*) internalGetResult { - return resultBackupSnapshot; -} -- (OWSSignalServiceProtosBackupSnapshotBuilder*) clear { - self.resultBackupSnapshot = [[OWSSignalServiceProtosBackupSnapshot alloc] init]; - return self; -} -- (OWSSignalServiceProtosBackupSnapshotBuilder*) clone { - return [OWSSignalServiceProtosBackupSnapshot builderWithPrototype:resultBackupSnapshot]; -} -- (OWSSignalServiceProtosBackupSnapshot*) defaultInstance { - return [OWSSignalServiceProtosBackupSnapshot defaultInstance]; -} -- (OWSSignalServiceProtosBackupSnapshot*) build { - [self checkInitialized]; - return [self buildPartial]; -} -- (OWSSignalServiceProtosBackupSnapshot*) buildPartial { - OWSSignalServiceProtosBackupSnapshot* returnMe = resultBackupSnapshot; - self.resultBackupSnapshot = nil; - return returnMe; -} -- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFrom:(OWSSignalServiceProtosBackupSnapshot*) other { - if (other == [OWSSignalServiceProtosBackupSnapshot defaultInstance]) { - return self; - } - if (other.entityArray.count > 0) { - if (resultBackupSnapshot.entityArray == nil) { - resultBackupSnapshot.entityArray = [[NSMutableArray alloc] initWithArray:other.entityArray]; - } else { - [resultBackupSnapshot.entityArray addObjectsFromArray:other.entityArray]; - } - } - [self mergeUnknownFields:other.unknownFields]; - return self; -} -- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input { - return [self mergeFromCodedInputStream:input extensionRegistry:[PBExtensionRegistry emptyRegistry]]; -} -- (OWSSignalServiceProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { - PBUnknownFieldSetBuilder* unknownFields = [PBUnknownFieldSet builderWithUnknownFields:self.unknownFields]; - while (YES) { - SInt32 tag = [input readTag]; - switch (tag) { - case 0: - [self setUnknownFields:[unknownFields build]]; - return self; - default: { - if (![self parseUnknownField:input unknownFields:unknownFields extensionRegistry:extensionRegistry tag:tag]) { - [self setUnknownFields:[unknownFields build]]; - return self; - } - break; - } - case 10: { - OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder* subBuilder = [OWSSignalServiceProtosBackupSnapshotBackupEntity builder]; - [input readMessage:subBuilder extensionRegistry:extensionRegistry]; - [self addEntity:[subBuilder buildPartial]]; - break; - } - } - } -} -- (NSMutableArray *)entity { - return resultBackupSnapshot.entityArray; -} -- (OWSSignalServiceProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index { - return [resultBackupSnapshot entityAtIndex:index]; -} -- (OWSSignalServiceProtosBackupSnapshotBuilder *)addEntity:(OWSSignalServiceProtosBackupSnapshotBackupEntity*)value { - if (resultBackupSnapshot.entityArray == nil) { - resultBackupSnapshot.entityArray = [[NSMutableArray alloc]init]; - } - [resultBackupSnapshot.entityArray addObject:value]; - return self; -} -- (OWSSignalServiceProtosBackupSnapshotBuilder *)setEntityArray:(NSArray *)array { - resultBackupSnapshot.entityArray = [[NSMutableArray alloc]initWithArray:array]; - return self; -} -- (OWSSignalServiceProtosBackupSnapshotBuilder *)clearEntity { - resultBackupSnapshot.entityArray = nil; - return self; -} -@end - // @@protoc_insertion_point(global_scope) diff --git a/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.h b/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.h new file mode 100644 index 0000000000..603d5e390f --- /dev/null +++ b/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.h @@ -0,0 +1,186 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! + +#import + +// @@protoc_insertion_point(imports) + +@class OWSSignaliOSProtosBackupSnapshot; +@class OWSSignaliOSProtosBackupSnapshotBackupEntity; +@class OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder; +@class OWSSignaliOSProtosBackupSnapshotBuilder; +@class ObjectiveCFileOptions; +@class ObjectiveCFileOptionsBuilder; +@class PBDescriptorProto; +@class PBDescriptorProtoBuilder; +@class PBDescriptorProtoExtensionRange; +@class PBDescriptorProtoExtensionRangeBuilder; +@class PBEnumDescriptorProto; +@class PBEnumDescriptorProtoBuilder; +@class PBEnumOptions; +@class PBEnumOptionsBuilder; +@class PBEnumValueDescriptorProto; +@class PBEnumValueDescriptorProtoBuilder; +@class PBEnumValueOptions; +@class PBEnumValueOptionsBuilder; +@class PBFieldDescriptorProto; +@class PBFieldDescriptorProtoBuilder; +@class PBFieldOptions; +@class PBFieldOptionsBuilder; +@class PBFileDescriptorProto; +@class PBFileDescriptorProtoBuilder; +@class PBFileDescriptorSet; +@class PBFileDescriptorSetBuilder; +@class PBFileOptions; +@class PBFileOptionsBuilder; +@class PBMessageOptions; +@class PBMessageOptionsBuilder; +@class PBMethodDescriptorProto; +@class PBMethodDescriptorProtoBuilder; +@class PBMethodOptions; +@class PBMethodOptionsBuilder; +@class PBOneofDescriptorProto; +@class PBOneofDescriptorProtoBuilder; +@class PBServiceDescriptorProto; +@class PBServiceDescriptorProtoBuilder; +@class PBServiceOptions; +@class PBServiceOptionsBuilder; +@class PBSourceCodeInfo; +@class PBSourceCodeInfoBuilder; +@class PBSourceCodeInfoLocation; +@class PBSourceCodeInfoLocationBuilder; +@class PBUninterpretedOption; +@class PBUninterpretedOptionBuilder; +@class PBUninterpretedOptionNamePart; +@class PBUninterpretedOptionNamePartBuilder; + + +typedef NS_ENUM(SInt32, OWSSignaliOSProtosBackupSnapshotBackupEntityType) { + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown = 0, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeMigration = 1, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeThread = 2, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeInteraction = 3, + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeAttachment = 4, +}; + +BOOL OWSSignaliOSProtosBackupSnapshotBackupEntityTypeIsValidValue(OWSSignaliOSProtosBackupSnapshotBackupEntityType value); +NSString *NSStringFromOWSSignaliOSProtosBackupSnapshotBackupEntityType(OWSSignaliOSProtosBackupSnapshotBackupEntityType value); + + +@interface OWSSignaliOSProtosOwssignaliOsprotosRoot : NSObject { +} ++ (PBExtensionRegistry*) extensionRegistry; ++ (void) registerAllExtensions:(PBMutableExtensionRegistry*) registry; +@end + +#define BackupSnapshot_entity @"entity" +@interface OWSSignaliOSProtosBackupSnapshot : PBGeneratedMessage { +@private + NSMutableArray * entityArray; +} +@property (readonly, strong) NSArray * entity; +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index; + ++ (instancetype) defaultInstance; +- (instancetype) defaultInstance; + +- (BOOL) isInitialized; +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output; +- (OWSSignaliOSProtosBackupSnapshotBuilder*) builder; ++ (OWSSignaliOSProtosBackupSnapshotBuilder*) builder; ++ (OWSSignaliOSProtosBackupSnapshotBuilder*) builderWithPrototype:(OWSSignaliOSProtosBackupSnapshot*) prototype; +- (OWSSignaliOSProtosBackupSnapshotBuilder*) toBuilder; + ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromData:(NSData*) data; ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input; ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input; ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; +@end + +#define BackupEntity_type @"type" +#define BackupEntity_entityData @"entityData" +@interface OWSSignaliOSProtosBackupSnapshotBackupEntity : PBGeneratedMessage { +@private + BOOL hasEntityData_:1; + BOOL hasType_:1; + NSData* entityData; + OWSSignaliOSProtosBackupSnapshotBackupEntityType type; +} +- (BOOL) hasType; +- (BOOL) hasEntityData; +@property (readonly) OWSSignaliOSProtosBackupSnapshotBackupEntityType type; +@property (readonly, strong) NSData* entityData; + ++ (instancetype) defaultInstance; +- (instancetype) defaultInstance; + +- (BOOL) isInitialized; +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builder; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builder; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builderWithPrototype:(OWSSignaliOSProtosBackupSnapshotBackupEntity*) prototype; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) toBuilder; + ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input; ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; +@end + +@interface OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder : PBGeneratedMessageBuilder { +@private + OWSSignaliOSProtosBackupSnapshotBackupEntity* resultBackupEntity; +} + +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) defaultInstance; + +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clear; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clone; + +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) build; +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) buildPartial; + +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFrom:(OWSSignaliOSProtosBackupSnapshotBackupEntity*) other; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; + +- (BOOL) hasType; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityType) type; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) setType:(OWSSignaliOSProtosBackupSnapshotBackupEntityType) value; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clearType; + +- (BOOL) hasEntityData; +- (NSData*) entityData; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) setEntityData:(NSData*) value; +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clearEntityData; +@end + +@interface OWSSignaliOSProtosBackupSnapshotBuilder : PBGeneratedMessageBuilder { +@private + OWSSignaliOSProtosBackupSnapshot* resultBackupSnapshot; +} + +- (OWSSignaliOSProtosBackupSnapshot*) defaultInstance; + +- (OWSSignaliOSProtosBackupSnapshotBuilder*) clear; +- (OWSSignaliOSProtosBackupSnapshotBuilder*) clone; + +- (OWSSignaliOSProtosBackupSnapshot*) build; +- (OWSSignaliOSProtosBackupSnapshot*) buildPartial; + +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFrom:(OWSSignaliOSProtosBackupSnapshot*) other; +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input; +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry; + +- (NSMutableArray *)entity; +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index; +- (OWSSignaliOSProtosBackupSnapshotBuilder *)addEntity:(OWSSignaliOSProtosBackupSnapshotBackupEntity*)value; +- (OWSSignaliOSProtosBackupSnapshotBuilder *)setEntityArray:(NSArray *)array; +- (OWSSignaliOSProtosBackupSnapshotBuilder *)clearEntity; +@end + + +// @@protoc_insertion_point(global_scope) diff --git a/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.m b/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.m new file mode 100644 index 0000000000..559fc36c05 --- /dev/null +++ b/SignalServiceKit/src/Protos/OWSSignaliOSProtos.pb.m @@ -0,0 +1,535 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! + +#import "OWSSignaliOSProtos.pb.h" +// @@protoc_insertion_point(imports) + +@implementation OWSSignaliOSProtosOwssignaliOsprotosRoot +static PBExtensionRegistry* extensionRegistry = nil; ++ (PBExtensionRegistry*) extensionRegistry { + return extensionRegistry; +} + ++ (void) initialize { + if (self == [OWSSignaliOSProtosOwssignaliOsprotosRoot class]) { + PBMutableExtensionRegistry* registry = [PBMutableExtensionRegistry registry]; + [self registerAllExtensions:registry]; + [ObjectivecDescriptorRoot registerAllExtensions:registry]; + extensionRegistry = registry; + } +} ++ (void) registerAllExtensions:(PBMutableExtensionRegistry*) registry { +} +@end + +@interface OWSSignaliOSProtosBackupSnapshot () +@property (strong) NSMutableArray * entityArray; +@end + +@implementation OWSSignaliOSProtosBackupSnapshot + +@synthesize entityArray; +@dynamic entity; +- (instancetype) init { + if ((self = [super init])) { + } + return self; +} +static OWSSignaliOSProtosBackupSnapshot* defaultOWSSignaliOSProtosBackupSnapshotInstance = nil; ++ (void) initialize { + if (self == [OWSSignaliOSProtosBackupSnapshot class]) { + defaultOWSSignaliOSProtosBackupSnapshotInstance = [[OWSSignaliOSProtosBackupSnapshot alloc] init]; + } +} ++ (instancetype) defaultInstance { + return defaultOWSSignaliOSProtosBackupSnapshotInstance; +} +- (instancetype) defaultInstance { + return defaultOWSSignaliOSProtosBackupSnapshotInstance; +} +- (NSArray *)entity { + return entityArray; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index { + return [entityArray objectAtIndex:index]; +} +- (BOOL) isInitialized { + return YES; +} +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output { + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignaliOSProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + [output writeMessage:1 value:element]; + }]; + [self.unknownFields writeToCodedOutputStream:output]; +} +- (SInt32) serializedSize { + __block SInt32 size_ = memoizedSerializedSize; + if (size_ != -1) { + return size_; + } + + size_ = 0; + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignaliOSProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + size_ += computeMessageSize(1, element); + }]; + size_ += self.unknownFields.serializedSize; + memoizedSerializedSize = size_; + return size_; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromData:(NSData*) data { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromData:data] build]; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromData:data extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromInputStream:input] build]; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromCodedInputStream:input] build]; +} ++ (OWSSignaliOSProtosBackupSnapshot*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshot*)[[[OWSSignaliOSProtosBackupSnapshot builder] mergeFromCodedInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBuilder*) builder { + return [[OWSSignaliOSProtosBackupSnapshotBuilder alloc] init]; +} ++ (OWSSignaliOSProtosBackupSnapshotBuilder*) builderWithPrototype:(OWSSignaliOSProtosBackupSnapshot*) prototype { + return [[OWSSignaliOSProtosBackupSnapshot builder] mergeFrom:prototype]; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) builder { + return [OWSSignaliOSProtosBackupSnapshot builder]; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) toBuilder { + return [OWSSignaliOSProtosBackupSnapshot builderWithPrototype:self]; +} +- (void) writeDescriptionTo:(NSMutableString*) output withIndent:(NSString*) indent { + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignaliOSProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + [output appendFormat:@"%@%@ {\n", indent, @"entity"]; + [element writeDescriptionTo:output + withIndent:[NSString stringWithFormat:@"%@ ", indent]]; + [output appendFormat:@"%@}\n", indent]; + }]; + [self.unknownFields writeDescriptionTo:output withIndent:indent]; +} +- (void) storeInDictionary:(NSMutableDictionary *)dictionary { + for (OWSSignaliOSProtosBackupSnapshotBackupEntity* element in self.entityArray) { + NSMutableDictionary *elementDictionary = [NSMutableDictionary dictionary]; + [element storeInDictionary:elementDictionary]; + [dictionary setObject:[NSDictionary dictionaryWithDictionary:elementDictionary] forKey:@"entity"]; + } + [self.unknownFields storeInDictionary:dictionary]; +} +- (BOOL) isEqual:(id)other { + if (other == self) { + return YES; + } + if (![other isKindOfClass:[OWSSignaliOSProtosBackupSnapshot class]]) { + return NO; + } + OWSSignaliOSProtosBackupSnapshot *otherMessage = other; + return + [self.entityArray isEqualToArray:otherMessage.entityArray] && + (self.unknownFields == otherMessage.unknownFields || (self.unknownFields != nil && [self.unknownFields isEqual:otherMessage.unknownFields])); +} +- (NSUInteger) hash { + __block NSUInteger hashCode = 7; + [self.entityArray enumerateObjectsUsingBlock:^(OWSSignaliOSProtosBackupSnapshotBackupEntity *element, NSUInteger idx, BOOL *stop) { + hashCode = hashCode * 31 + [element hash]; + }]; + hashCode = hashCode * 31 + [self.unknownFields hash]; + return hashCode; +} +@end + +@interface OWSSignaliOSProtosBackupSnapshotBackupEntity () +@property OWSSignaliOSProtosBackupSnapshotBackupEntityType type; +@property (strong) NSData* entityData; +@end + +@implementation OWSSignaliOSProtosBackupSnapshotBackupEntity + +- (BOOL) hasType { + return !!hasType_; +} +- (void) setHasType:(BOOL) _value_ { + hasType_ = !!_value_; +} +@synthesize type; +- (BOOL) hasEntityData { + return !!hasEntityData_; +} +- (void) setHasEntityData:(BOOL) _value_ { + hasEntityData_ = !!_value_; +} +@synthesize entityData; +- (instancetype) init { + if ((self = [super init])) { + self.type = OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown; + self.entityData = [NSData data]; + } + return self; +} +static OWSSignaliOSProtosBackupSnapshotBackupEntity* defaultOWSSignaliOSProtosBackupSnapshotBackupEntityInstance = nil; ++ (void) initialize { + if (self == [OWSSignaliOSProtosBackupSnapshotBackupEntity class]) { + defaultOWSSignaliOSProtosBackupSnapshotBackupEntityInstance = [[OWSSignaliOSProtosBackupSnapshotBackupEntity alloc] init]; + } +} ++ (instancetype) defaultInstance { + return defaultOWSSignaliOSProtosBackupSnapshotBackupEntityInstance; +} +- (instancetype) defaultInstance { + return defaultOWSSignaliOSProtosBackupSnapshotBackupEntityInstance; +} +- (BOOL) isInitialized { + return YES; +} +- (void) writeToCodedOutputStream:(PBCodedOutputStream*) output { + if (self.hasType) { + [output writeEnum:1 value:self.type]; + } + if (self.hasEntityData) { + [output writeData:2 value:self.entityData]; + } + [self.unknownFields writeToCodedOutputStream:output]; +} +- (SInt32) serializedSize { + __block SInt32 size_ = memoizedSerializedSize; + if (size_ != -1) { + return size_; + } + + size_ = 0; + if (self.hasType) { + size_ += computeEnumSize(1, self.type); + } + if (self.hasEntityData) { + size_ += computeDataSize(2, self.entityData); + } + size_ += self.unknownFields.serializedSize; + memoizedSerializedSize = size_; + return size_; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromData:data] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromData:(NSData*) data extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromData:data extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromInputStream:input] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromInputStream:(NSInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromCodedInputStream:input] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntity*) parseFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + return (OWSSignaliOSProtosBackupSnapshotBackupEntity*)[[[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFromCodedInputStream:input extensionRegistry:extensionRegistry] build]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builder { + return [[OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder alloc] init]; +} ++ (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builderWithPrototype:(OWSSignaliOSProtosBackupSnapshotBackupEntity*) prototype { + return [[OWSSignaliOSProtosBackupSnapshotBackupEntity builder] mergeFrom:prototype]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) builder { + return [OWSSignaliOSProtosBackupSnapshotBackupEntity builder]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) toBuilder { + return [OWSSignaliOSProtosBackupSnapshotBackupEntity builderWithPrototype:self]; +} +- (void) writeDescriptionTo:(NSMutableString*) output withIndent:(NSString*) indent { + if (self.hasType) { + [output appendFormat:@"%@%@: %@\n", indent, @"type", NSStringFromOWSSignaliOSProtosBackupSnapshotBackupEntityType(self.type)]; + } + if (self.hasEntityData) { + [output appendFormat:@"%@%@: %@\n", indent, @"entityData", self.entityData]; + } + [self.unknownFields writeDescriptionTo:output withIndent:indent]; +} +- (void) storeInDictionary:(NSMutableDictionary *)dictionary { + if (self.hasType) { + [dictionary setObject: @(self.type) forKey: @"type"]; + } + if (self.hasEntityData) { + [dictionary setObject: self.entityData forKey: @"entityData"]; + } + [self.unknownFields storeInDictionary:dictionary]; +} +- (BOOL) isEqual:(id)other { + if (other == self) { + return YES; + } + if (![other isKindOfClass:[OWSSignaliOSProtosBackupSnapshotBackupEntity class]]) { + return NO; + } + OWSSignaliOSProtosBackupSnapshotBackupEntity *otherMessage = other; + return + self.hasType == otherMessage.hasType && + (!self.hasType || self.type == otherMessage.type) && + self.hasEntityData == otherMessage.hasEntityData && + (!self.hasEntityData || [self.entityData isEqual:otherMessage.entityData]) && + (self.unknownFields == otherMessage.unknownFields || (self.unknownFields != nil && [self.unknownFields isEqual:otherMessage.unknownFields])); +} +- (NSUInteger) hash { + __block NSUInteger hashCode = 7; + if (self.hasType) { + hashCode = hashCode * 31 + self.type; + } + if (self.hasEntityData) { + hashCode = hashCode * 31 + [self.entityData hash]; + } + hashCode = hashCode * 31 + [self.unknownFields hash]; + return hashCode; +} +@end + +BOOL OWSSignaliOSProtosBackupSnapshotBackupEntityTypeIsValidValue(OWSSignaliOSProtosBackupSnapshotBackupEntityType value) { + switch (value) { + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown: + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeMigration: + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeThread: + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeInteraction: + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeAttachment: + return YES; + default: + return NO; + } +} +NSString *NSStringFromOWSSignaliOSProtosBackupSnapshotBackupEntityType(OWSSignaliOSProtosBackupSnapshotBackupEntityType value) { + switch (value) { + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown: + return @"OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown"; + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeMigration: + return @"OWSSignaliOSProtosBackupSnapshotBackupEntityTypeMigration"; + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeThread: + return @"OWSSignaliOSProtosBackupSnapshotBackupEntityTypeThread"; + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeInteraction: + return @"OWSSignaliOSProtosBackupSnapshotBackupEntityTypeInteraction"; + case OWSSignaliOSProtosBackupSnapshotBackupEntityTypeAttachment: + return @"OWSSignaliOSProtosBackupSnapshotBackupEntityTypeAttachment"; + default: + return nil; + } +} + +@interface OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder() +@property (strong) OWSSignaliOSProtosBackupSnapshotBackupEntity* resultBackupEntity; +@end + +@implementation OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder +@synthesize resultBackupEntity; +- (instancetype) init { + if ((self = [super init])) { + self.resultBackupEntity = [[OWSSignaliOSProtosBackupSnapshotBackupEntity alloc] init]; + } + return self; +} +- (PBGeneratedMessage*) internalGetResult { + return resultBackupEntity; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clear { + self.resultBackupEntity = [[OWSSignaliOSProtosBackupSnapshotBackupEntity alloc] init]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clone { + return [OWSSignaliOSProtosBackupSnapshotBackupEntity builderWithPrototype:resultBackupEntity]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) defaultInstance { + return [OWSSignaliOSProtosBackupSnapshotBackupEntity defaultInstance]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) build { + [self checkInitialized]; + return [self buildPartial]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*) buildPartial { + OWSSignaliOSProtosBackupSnapshotBackupEntity* returnMe = resultBackupEntity; + self.resultBackupEntity = nil; + return returnMe; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFrom:(OWSSignaliOSProtosBackupSnapshotBackupEntity*) other { + if (other == [OWSSignaliOSProtosBackupSnapshotBackupEntity defaultInstance]) { + return self; + } + if (other.hasType) { + [self setType:other.type]; + } + if (other.hasEntityData) { + [self setEntityData:other.entityData]; + } + [self mergeUnknownFields:other.unknownFields]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input { + return [self mergeFromCodedInputStream:input extensionRegistry:[PBExtensionRegistry emptyRegistry]]; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + PBUnknownFieldSetBuilder* unknownFields = [PBUnknownFieldSet builderWithUnknownFields:self.unknownFields]; + while (YES) { + SInt32 tag = [input readTag]; + switch (tag) { + case 0: + [self setUnknownFields:[unknownFields build]]; + return self; + default: { + if (![self parseUnknownField:input unknownFields:unknownFields extensionRegistry:extensionRegistry tag:tag]) { + [self setUnknownFields:[unknownFields build]]; + return self; + } + break; + } + case 8: { + OWSSignaliOSProtosBackupSnapshotBackupEntityType value = (OWSSignaliOSProtosBackupSnapshotBackupEntityType)[input readEnum]; + if (OWSSignaliOSProtosBackupSnapshotBackupEntityTypeIsValidValue(value)) { + [self setType:value]; + } else { + [unknownFields mergeVarintField:1 value:value]; + } + break; + } + case 18: { + [self setEntityData:[input readData]]; + break; + } + } + } +} +- (BOOL) hasType { + return resultBackupEntity.hasType; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityType) type { + return resultBackupEntity.type; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) setType:(OWSSignaliOSProtosBackupSnapshotBackupEntityType) value { + resultBackupEntity.hasType = YES; + resultBackupEntity.type = value; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clearType { + resultBackupEntity.hasType = NO; + resultBackupEntity.type = OWSSignaliOSProtosBackupSnapshotBackupEntityTypeUnknown; + return self; +} +- (BOOL) hasEntityData { + return resultBackupEntity.hasEntityData; +} +- (NSData*) entityData { + return resultBackupEntity.entityData; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) setEntityData:(NSData*) value { + resultBackupEntity.hasEntityData = YES; + resultBackupEntity.entityData = value; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder*) clearEntityData { + resultBackupEntity.hasEntityData = NO; + resultBackupEntity.entityData = [NSData data]; + return self; +} +@end + +@interface OWSSignaliOSProtosBackupSnapshotBuilder() +@property (strong) OWSSignaliOSProtosBackupSnapshot* resultBackupSnapshot; +@end + +@implementation OWSSignaliOSProtosBackupSnapshotBuilder +@synthesize resultBackupSnapshot; +- (instancetype) init { + if ((self = [super init])) { + self.resultBackupSnapshot = [[OWSSignaliOSProtosBackupSnapshot alloc] init]; + } + return self; +} +- (PBGeneratedMessage*) internalGetResult { + return resultBackupSnapshot; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) clear { + self.resultBackupSnapshot = [[OWSSignaliOSProtosBackupSnapshot alloc] init]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) clone { + return [OWSSignaliOSProtosBackupSnapshot builderWithPrototype:resultBackupSnapshot]; +} +- (OWSSignaliOSProtosBackupSnapshot*) defaultInstance { + return [OWSSignaliOSProtosBackupSnapshot defaultInstance]; +} +- (OWSSignaliOSProtosBackupSnapshot*) build { + [self checkInitialized]; + return [self buildPartial]; +} +- (OWSSignaliOSProtosBackupSnapshot*) buildPartial { + OWSSignaliOSProtosBackupSnapshot* returnMe = resultBackupSnapshot; + self.resultBackupSnapshot = nil; + return returnMe; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFrom:(OWSSignaliOSProtosBackupSnapshot*) other { + if (other == [OWSSignaliOSProtosBackupSnapshot defaultInstance]) { + return self; + } + if (other.entityArray.count > 0) { + if (resultBackupSnapshot.entityArray == nil) { + resultBackupSnapshot.entityArray = [[NSMutableArray alloc] initWithArray:other.entityArray]; + } else { + [resultBackupSnapshot.entityArray addObjectsFromArray:other.entityArray]; + } + } + [self mergeUnknownFields:other.unknownFields]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input { + return [self mergeFromCodedInputStream:input extensionRegistry:[PBExtensionRegistry emptyRegistry]]; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder*) mergeFromCodedInputStream:(PBCodedInputStream*) input extensionRegistry:(PBExtensionRegistry*) extensionRegistry { + PBUnknownFieldSetBuilder* unknownFields = [PBUnknownFieldSet builderWithUnknownFields:self.unknownFields]; + while (YES) { + SInt32 tag = [input readTag]; + switch (tag) { + case 0: + [self setUnknownFields:[unknownFields build]]; + return self; + default: { + if (![self parseUnknownField:input unknownFields:unknownFields extensionRegistry:extensionRegistry tag:tag]) { + [self setUnknownFields:[unknownFields build]]; + return self; + } + break; + } + case 10: { + OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder* subBuilder = [OWSSignaliOSProtosBackupSnapshotBackupEntity builder]; + [input readMessage:subBuilder extensionRegistry:extensionRegistry]; + [self addEntity:[subBuilder buildPartial]]; + break; + } + } + } +} +- (NSMutableArray *)entity { + return resultBackupSnapshot.entityArray; +} +- (OWSSignaliOSProtosBackupSnapshotBackupEntity*)entityAtIndex:(NSUInteger)index { + return [resultBackupSnapshot entityAtIndex:index]; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder *)addEntity:(OWSSignaliOSProtosBackupSnapshotBackupEntity*)value { + if (resultBackupSnapshot.entityArray == nil) { + resultBackupSnapshot.entityArray = [[NSMutableArray alloc]init]; + } + [resultBackupSnapshot.entityArray addObject:value]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder *)setEntityArray:(NSArray *)array { + resultBackupSnapshot.entityArray = [[NSMutableArray alloc]initWithArray:array]; + return self; +} +- (OWSSignaliOSProtosBackupSnapshotBuilder *)clearEntity { + resultBackupSnapshot.entityArray = nil; + return self; +} +@end + + +// @@protoc_insertion_point(global_scope) From 5c3bc74d06c8dff92e744075f02158bd0e155077 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Tue, 20 Mar 2018 11:08:13 -0400 Subject: [PATCH 11/13] Move backup protos to their own proto schema. --- Pods | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pods b/Pods index 39c23000fa..0e1c47543a 160000 --- a/Pods +++ b/Pods @@ -1 +1 @@ -Subproject commit 39c23000fa4c377adddcd952b8a8093a14ade407 +Subproject commit 0e1c47543afd02421f95488463c64b0e52892333 From 34d79265a193fac7791fb7de4e30fd13654bad34 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Tue, 20 Mar 2018 11:23:45 -0400 Subject: [PATCH 12/13] Respond to CR. --- Signal/src/util/OWSBackupAPI.swift | 5 +--- Signal/src/util/OWSBackupExportJob.m | 44 ++++++++++++++++------------ Signal/src/util/OWSBackupIO.m | 14 +++++---- Signal/src/util/OWSBackupImportJob.m | 6 ++-- 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/Signal/src/util/OWSBackupAPI.swift b/Signal/src/util/OWSBackupAPI.swift index ad67f410e3..be943ff958 100644 --- a/Signal/src/util/OWSBackupAPI.swift +++ b/Signal/src/util/OWSBackupAPI.swift @@ -239,10 +239,7 @@ import CloudKit success: @escaping (()) -> Void, failure: @escaping (Error) -> Void) { - var recordIDs = [CKRecordID]() - for recordName in recordNames { - recordIDs.append(CKRecordID(recordName: recordName)) - } + let recordIDs = recordNames.map { CKRecordID(recordName: $0) } let deleteOperation = CKModifyRecordsOperation(recordsToSave: nil, recordIDsToDelete: recordIDs) deleteOperation.modifyRecordsCompletionBlock = { (records, recordIds, error) in diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index 44431e486c..f96e5585dd 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -65,7 +65,7 @@ NS_ASSUME_NONNULL_BEGIN // Writes db entities using protobufs into snapshot fragments. // Snapshot fragments are compressed (they compress _very well_, // around 20x smaller) then encrypted. Ordering matters in -// snapshot contents (entities should we restored in the same +// snapshot contents (entities should be restored in the same // order they are serialized), so we are always careful to preserve // ordering of entities within a snapshot AND ordering of snapshot // fragments within a bakckup. @@ -80,7 +80,7 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic) NSMutableArray *exportItems; -@property (nonatomic, nullable) OWSSignalServiceProtosBackupSnapshotBuilder *backupSnapshotBuilder; +@property (nonatomic, nullable) OWSSignaliOSProtosBackupSnapshotBuilder *backupSnapshotBuilder; @property (nonatomic) NSUInteger cachedItemCount; @@ -113,7 +113,7 @@ NS_ASSUME_NONNULL_BEGIN // use this state), but I think it'll be helpful to have around to future-proof // this work, help with debugging issue, etc. - (BOOL)writeObject:(TSYapDatabaseObject *)object - entityType:(OWSSignalServiceProtosBackupSnapshotBackupEntityType)entityType + entityType:(OWSSignaliOSProtosBackupSnapshotBackupEntityType)entityType { OWSAssert(object); @@ -124,11 +124,11 @@ NS_ASSUME_NONNULL_BEGIN } if (!self.backupSnapshotBuilder) { - self.backupSnapshotBuilder = [OWSSignalServiceProtosBackupSnapshotBuilder new]; + self.backupSnapshotBuilder = [OWSSignaliOSProtosBackupSnapshotBuilder new]; } - OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder *entityBuilder = - [OWSSignalServiceProtosBackupSnapshotBackupEntityBuilder new]; + OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder *entityBuilder = + [OWSSignaliOSProtosBackupSnapshotBackupEntityBuilder new]; [entityBuilder setType:entityType]; [entityBuilder setEntityData:data]; @@ -426,12 +426,12 @@ NS_ASSUME_NONNULL_BEGIN NSString *, Class, EntityFilter _Nullable, - OWSSignalServiceProtosBackupSnapshotBackupEntityType); + OWSSignaliOSProtosBackupSnapshotBackupEntityType); ExportBlock exportEntities = ^(YapDatabaseReadTransaction *transaction, NSString *collection, Class expectedClass, EntityFilter _Nullable filter, - OWSSignalServiceProtosBackupSnapshotBackupEntityType entityType) { + OWSSignaliOSProtosBackupSnapshotBackupEntityType entityType) { __block NSUInteger count = 0; [transaction enumerateKeysAndObjectsInCollection:collection @@ -469,7 +469,7 @@ NS_ASSUME_NONNULL_BEGIN [TSThread collection], [TSThread class], nil, - OWSSignalServiceProtosBackupSnapshotBackupEntityTypeThread); + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeThread); if (aborted) { return; } @@ -499,7 +499,7 @@ NS_ASSUME_NONNULL_BEGIN return YES; }, - OWSSignalServiceProtosBackupSnapshotBackupEntityTypeAttachment); + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeAttachment); if (aborted) { return; } @@ -523,7 +523,7 @@ NS_ASSUME_NONNULL_BEGIN } return YES; }, - OWSSignalServiceProtosBackupSnapshotBackupEntityTypeInteraction); + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeInteraction); if (aborted) { return; } @@ -532,7 +532,7 @@ NS_ASSUME_NONNULL_BEGIN [OWSDatabaseMigration collection], [OWSDatabaseMigration class], nil, - OWSSignalServiceProtosBackupSnapshotBackupEntityTypeMigration); + OWSSignaliOSProtosBackupSnapshotBackupEntityTypeMigration); }]; if (aborted || self.isComplete) { @@ -568,28 +568,36 @@ NS_ASSUME_NONNULL_BEGIN self.savedDatabaseItems = [NSMutableArray new]; self.savedAttachmentItems = [NSMutableArray new]; + unsigned long long totalFileSize = 0; + NSUInteger totalFileCount = 0; { - unsigned long long totalFileSize = 0; + unsigned long long databaseFileSize = 0; for (OWSBackupExportItem *item in self.unsavedDatabaseItems) { - totalFileSize += [OWSFileSystem fileSizeOfPath:item.encryptedItem.filePath].unsignedLongLongValue; + databaseFileSize += [OWSFileSystem fileSizeOfPath:item.encryptedItem.filePath].unsignedLongLongValue; } DDLogInfo(@"%@ exporting %@: count: %zd, bytes: %llu.", self.logTag, @"database items", self.unsavedDatabaseItems.count, - totalFileSize); + databaseFileSize); + totalFileSize += databaseFileSize; + totalFileCount += self.unsavedDatabaseItems.count; } { - unsigned long long totalFileSize = 0; + unsigned long long attachmentFileSize = 0; for (OWSAttachmentExport *attachmentExport in self.unsavedAttachmentExports) { - totalFileSize += [OWSFileSystem fileSizeOfPath:attachmentExport.attachmentFilePath].unsignedLongLongValue; + attachmentFileSize += + [OWSFileSystem fileSizeOfPath:attachmentExport.attachmentFilePath].unsignedLongLongValue; } DDLogInfo(@"%@ exporting %@: count: %zd, bytes: %llu.", self.logTag, @"attachment items", self.unsavedAttachmentExports.count, - totalFileSize); + attachmentFileSize); + totalFileSize += attachmentFileSize; + totalFileCount += self.unsavedAttachmentExports.count; } + DDLogInfo(@"%@ exporting %@: count: %zd, bytes: %llu.", self.logTag, @"all items", totalFileCount, totalFileSize); [self saveNextFileToCloudWithCompletion:completion]; } diff --git a/Signal/src/util/OWSBackupIO.m b/Signal/src/util/OWSBackupIO.m index ebf8f59858..02b50d9fb7 100644 --- a/Signal/src/util/OWSBackupIO.m +++ b/Signal/src/util/OWSBackupIO.m @@ -13,6 +13,10 @@ NS_ASSUME_NONNULL_BEGIN // TODO: static const NSUInteger kOWSBackupKeyLength = 32; +// LZMA algorithm significantly outperforms the other compressionlib options +// for our database snapshots and is a widely adopted standard. +static const compression_algorithm SignalCompressionAlgorithm = COMPRESSION_LZMA; + @implementation OWSBackupEncryptedItem @end @@ -196,9 +200,8 @@ static const NSUInteger kOWSBackupKeyLength = 32; if (!dstBuffer) { return nil; } - // TODO: Should we use COMPRESSION_LZFSE? - size_t dstLength - = compression_encode_buffer(dstBuffer, dstBufferLength, srcBuffer, srcLength, NULL, COMPRESSION_LZFSE); + size_t dstLength = compression_encode_buffer( + dstBuffer, dstBufferLength, srcBuffer, srcLength, NULL, SignalCompressionAlgorithm); NSData *compressedData = [NSData dataWithBytesNoCopy:dstBuffer length:dstLength freeWhenDone:YES]; DDLogVerbose(@"%@ compressed %zd -> %zd = %0.2f", @@ -233,9 +236,8 @@ static const NSUInteger kOWSBackupKeyLength = 32; if (!dstBuffer) { return nil; } - // TODO: Should we use COMPRESSION_LZFSE? - size_t dstLength - = compression_decode_buffer(dstBuffer, dstBufferLength, srcBuffer, srcLength, NULL, COMPRESSION_LZFSE); + size_t dstLength = compression_decode_buffer( + dstBuffer, dstBufferLength, srcBuffer, srcLength, NULL, SignalCompressionAlgorithm); NSData *decompressedData = [NSData dataWithBytesNoCopy:dstBuffer length:dstLength freeWhenDone:YES]; OWSAssert(decompressedData.length == uncompressedDataLength); DDLogVerbose(@"%@ decompressed %zd -> %zd = %0.2f", diff --git a/Signal/src/util/OWSBackupImportJob.m b/Signal/src/util/OWSBackupImportJob.m index 99e7cda03f..76171cc3f8 100644 --- a/Signal/src/util/OWSBackupImportJob.m +++ b/Signal/src/util/OWSBackupImportJob.m @@ -515,15 +515,15 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe aborted = YES; return completion(NO); } - OWSSignalServiceProtosBackupSnapshot *_Nullable entities = - [OWSSignalServiceProtosBackupSnapshot parseFromData:uncompressedData]; + OWSSignaliOSProtosBackupSnapshot *_Nullable entities = + [OWSSignaliOSProtosBackupSnapshot parseFromData:uncompressedData]; if (!entities || entities.entity.count < 1) { DDLogError(@"%@ missing entities.", self.logTag); // Database-related errors are unrecoverable. aborted = YES; return completion(NO); } - for (OWSSignalServiceProtosBackupSnapshotBackupEntity *entity in entities.entity) { + for (OWSSignaliOSProtosBackupSnapshotBackupEntity *entity in entities.entity) { NSData *_Nullable entityData = entity.entityData; if (entityData.length < 1) { DDLogError(@"%@ missing entity data.", self.logTag); From e8a716f2b4e1a1f9d6f3bdc9b6e4766151d23914 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Tue, 20 Mar 2018 11:25:00 -0400 Subject: [PATCH 13/13] Update cocoapods. --- Pods | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pods b/Pods index 0e1c47543a..5564eb7e18 160000 --- a/Pods +++ b/Pods @@ -1 +1 @@ -Subproject commit 0e1c47543afd02421f95488463c64b0e52892333 +Subproject commit 5564eb7e1870233872738ab652793883d1dc1c3d