diff --git a/Signal/src/ViewControllers/AppSettings/OWSBackupSettingsViewController.m b/Signal/src/ViewControllers/AppSettings/OWSBackupSettingsViewController.m index afe7a25e43..a2569aa2cc 100644 --- a/Signal/src/ViewControllers/AppSettings/OWSBackupSettingsViewController.m +++ b/Signal/src/ViewControllers/AppSettings/OWSBackupSettingsViewController.m @@ -70,9 +70,81 @@ NS_ASSUME_NONNULL_BEGIN selector:@selector(isBackupEnabledDidChange:)]]; [contents addSection:enableSection]; + if (isBackupEnabled) { + // TODO: This UI is temporary. + // Enabling backup will involve entering and registering a PIN. + OWSTableSection *progressSection = [OWSTableSection new]; + [progressSection + addItem:[OWSTableItem labelItemWithText:NSLocalizedString(@"SETTINGS_BACKUP_STATUS", + @"Label for status row in the in the backup settings view.") + accessoryText:[self backupExportStateLocalizedDescription]]]; + if (OWSBackup.sharedManager.backupExportState == OWSBackupState_InProgress) { + if (OWSBackup.sharedManager.backupExportDescription) { + [progressSection + addItem:[OWSTableItem + labelItemWithText:NSLocalizedString(@"SETTINGS_BACKUP_PHASE", + @"Label for phase row in the in the backup settings view.") + accessoryText:OWSBackup.sharedManager.backupExportDescription]]; + if (OWSBackup.sharedManager.backupExportProgress) { + NSUInteger progressPercent + = (NSUInteger)round(OWSBackup.sharedManager.backupExportProgress.floatValue * 100); + NSNumberFormatter *numberFormatter = [[NSNumberFormatter alloc] init]; + [numberFormatter setNumberStyle:NSNumberFormatterPercentStyle]; + [numberFormatter setMaximumFractionDigits:0]; + [numberFormatter setMultiplier:@1]; + NSString *progressString = [numberFormatter stringFromNumber:@(progressPercent)]; + [progressSection + addItem:[OWSTableItem + labelItemWithText:NSLocalizedString(@"SETTINGS_BACKUP_PROGRESS", + @"Label for phase row in the in the backup settings view.") + accessoryText:progressString]]; + } + } + } + + switch (OWSBackup.sharedManager.backupExportState) { + case OWSBackupState_Idle: + case OWSBackupState_Failed: + case OWSBackupState_Succeeded: + [progressSection + addItem:[OWSTableItem disclosureItemWithText: + NSLocalizedString(@"SETTINGS_BACKUP_BACKUP_NOW", + @"Label for 'backup now' button in the backup settings view.") + actionBlock:^{ + [OWSBackup.sharedManager tryToExportBackup]; + }]]; + break; + case OWSBackupState_InProgress: + [progressSection + addItem:[OWSTableItem disclosureItemWithText: + NSLocalizedString(@"SETTINGS_BACKUP_CANCEL_BACKUP", + @"Label for 'cancel backup' button in the backup settings view.") + actionBlock:^{ + [OWSBackup.sharedManager cancelExportBackup]; + }]]; + break; + } + + [contents addSection:progressSection]; + } + self.contents = contents; } +- (NSString *)backupExportStateLocalizedDescription +{ + switch (OWSBackup.sharedManager.backupExportState) { + case OWSBackupState_Idle: + return NSLocalizedString(@"SETTINGS_BACKUP_STATUS_IDLE", @"Indicates that app is not backing up."); + case OWSBackupState_InProgress: + return NSLocalizedString(@"SETTINGS_BACKUP_STATUS_IN_PROGRESS", @"Indicates that app is backing up."); + case OWSBackupState_Failed: + return NSLocalizedString(@"SETTINGS_BACKUP_STATUS_FAILED", @"Indicates that the last backup failed."); + case OWSBackupState_Succeeded: + return NSLocalizedString(@"SETTINGS_BACKUP_STATUS_SUCCEEDED", @"Indicates that the last backup succeeded."); + } +} + - (void)isBackupEnabledDidChange:(UISwitch *)sender { [OWSBackup.sharedManager setIsBackupEnabled:sender.isOn]; diff --git a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m index e55826d4e0..ee9d39b33a 100644 --- a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m +++ b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m @@ -90,7 +90,20 @@ NS_ASSUME_NONNULL_BEGIN { DDLogInfo(@"%@ tryToImportBackup.", self.logTag); - [OWSBackup.sharedManager tryToImportBackup]; + UIAlertController *controller = + [UIAlertController alertControllerWithTitle:@"Restore CloudKit Backup" + message:@"This will delete all of your database contents." + preferredStyle:UIAlertControllerStyleAlert]; + + [controller addAction:[UIAlertAction + actionWithTitle:@"Restore" + style:UIAlertActionStyleDefault + handler:^(UIAlertAction *_Nonnull action) { + [OWSBackup.sharedManager tryToImportBackup]; + }]]; + [controller addAction:[OWSAlerts cancelAction]]; + UIViewController *fromViewController = [[UIApplication sharedApplication] frontmostViewController]; + [fromViewController presentViewController:controller animated:YES completion:nil]; } @end diff --git a/Signal/src/util/OWSBackup.h b/Signal/src/util/OWSBackup.h index b1f0bf0563..5327f78d8b 100644 --- a/Signal/src/util/OWSBackup.h +++ b/Signal/src/util/OWSBackup.h @@ -33,6 +33,7 @@ typedef NS_ENUM(NSUInteger, OWSBackupState) { #pragma mark - Backup Export @property (nonatomic, readonly) OWSBackupState backupExportState; + // If a "backup export" is in progress (see backupExportState), // backupExportDescription _might_ contain a string that describes // the current phase and backupExportProgress _might_ contain a @@ -44,9 +45,13 @@ typedef NS_ENUM(NSUInteger, OWSBackupState) { - (BOOL)isBackupEnabled; - (void)setIsBackupEnabled:(BOOL)value; +- (void)tryToExportBackup; +- (void)cancelExportBackup; + #pragma mark - Backup Import @property (nonatomic, readonly) OWSBackupState backupImportState; + // If a "backup import" is in progress (see backupImportState), // backupImportDescription _might_ contain a string that describes // the current phase and backupImportProgress _might_ contain a diff --git a/Signal/src/util/OWSBackup.m b/Signal/src/util/OWSBackup.m index 645807a16d..d05368211b 100644 --- a/Signal/src/util/OWSBackup.m +++ b/Signal/src/util/OWSBackup.m @@ -134,6 +134,39 @@ NS_ASSUME_NONNULL_BEGIN #pragma mark - Backup Export +- (void)tryToExportBackup +{ + OWSAssertIsOnMainThread(); + OWSAssert(!self.backupExportJob); + + if (!self.canBackupExport) { + // TODO: Offer a reason in the UI. + return; + } + + // In development, make sure there's no export or import in progress. + [self.backupExportJob cancel]; + self.backupExportJob = nil; + [self.backupImportJob cancel]; + self.backupImportJob = nil; + + _backupExportState = OWSBackupState_InProgress; + + self.backupExportJob = + [[OWSBackupExportJob alloc] initWithDelegate:self primaryStorage:[OWSPrimaryStorage sharedManager]]; + [self.backupExportJob startAsync]; + + [self postDidChangeNotification]; +} + +- (void)cancelExportBackup +{ + [self.backupExportJob cancel]; + self.backupExportJob = nil; + + [self ensureBackupExportState]; +} + - (void)setLastExportSuccessDate:(NSDate *)value { OWSAssert(value); @@ -190,7 +223,7 @@ NS_ASSUME_NONNULL_BEGIN [self ensureBackupExportState]; } -- (BOOL)shouldHaveBackupExport +- (BOOL)canBackupExport { if (!self.isBackupEnabled) { return NO; @@ -202,19 +235,27 @@ NS_ASSUME_NONNULL_BEGIN if (![TSAccountManager isRegistered]) { return NO; } + return YES; +} + +- (BOOL)shouldHaveBackupExport +{ + if (!self.canBackupExport) { + return NO; + } + if (self.backupExportJob) { + // If there's already a job in progress, let it complete. + return YES; + } NSDate *_Nullable lastExportSuccessDate = self.lastExportSuccessDate; NSDate *_Nullable lastExportFailureDate = self.lastExportFailureDate; // Wait N hours before retrying after a success. const NSTimeInterval kRetryAfterSuccess = 24 * kHourInterval; - // TODO: Remove. - // const NSTimeInterval kRetryAfterSuccess = 0; if (lastExportSuccessDate && fabs(lastExportSuccessDate.timeIntervalSinceNow) < kRetryAfterSuccess) { return NO; } // Wait N hours before retrying after a failure. const NSTimeInterval kRetryAfterFailure = 6 * kHourInterval; - // TODO: Remove. - // const NSTimeInterval kRetryAfterFailure = 0; if (lastExportFailureDate && fabs(lastExportFailureDate.timeIntervalSinceNow) < kRetryAfterFailure) { return NO; } diff --git a/Signal/src/util/OWSBackupAPI.swift b/Signal/src/util/OWSBackupAPI.swift index f5b375339f..79344a34e1 100644 --- a/Signal/src/util/OWSBackupAPI.swift +++ b/Signal/src/util/OWSBackupAPI.swift @@ -13,12 +13,26 @@ import CloudKit static let signalBackupRecordType = "signalBackup" static let manifestRecordName = "manifest" static let payloadKey = "payload" + static let maxRetries = 5 - @objc - public class func recordIdForTest() -> String { + private class func recordIdForTest() -> String { return "test-\(NSUUID().uuidString)" } + private class func database() -> CKDatabase { + let myContainer = CKContainer.default() + let privateDatabase = myContainer.privateCloudDatabase + return privateDatabase + } + + private class func invalidServiceResponseError() -> Error { + return OWSErrorWithCodeDescription(.backupFailure, + NSLocalizedString("BACKUP_EXPORT_ERROR_INVALID_CLOUDKIT_RESPONSE", + comment: "Error indicating that the app received an invalid response from CloudKit.")) + } + + // MARK: - Upload + @objc public class func saveTestFileToCloud(fileUrl: URL, success: @escaping (String) -> Void, @@ -40,9 +54,9 @@ import CloudKit failure: @escaping (Error) -> Void) { saveFileToCloud(fileUrl: fileUrl, recordName: "ephemeralFile-\(NSUUID().uuidString)", - recordType: signalBackupRecordType, - success: success, - failure: failure) + recordType: signalBackupRecordType, + success: success, + failure: failure) } // "Persistent" files may be shared between backup export; they should only be saved @@ -92,47 +106,46 @@ import CloudKit public class func saveRecordToCloud(record: CKRecord, success: @escaping (String) -> Void, failure: @escaping (Error) -> Void) { - - let myContainer = CKContainer.default() - let privateDatabase = myContainer.privateCloudDatabase - privateDatabase.save(record) { - (record, error) in - - if let error = error { - Logger.error("\(self.logTag) error saving record: \(error)") - failure(error) - } else { - guard let recordName = record?.recordID.recordName else { - Logger.error("\(self.logTag) error retrieving saved record's name.") - failure(OWSErrorWithCodeDescription(.exportBackupError, - NSLocalizedString("BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED", - comment: "Error indicating the a backup export failed to save a file to the cloud."))) - return - } - Logger.info("\(self.logTag) saved record.") - success(recordName) - } - } + saveRecordToCloud(record: record, + remainingRetries: maxRetries, + success: success, + failure: failure) } - @objc - public class func deleteRecordFromCloud(recordName: String, - success: @escaping (()) -> Void, - failure: @escaping (Error) -> Void) { + private class func saveRecordToCloud(record: CKRecord, + remainingRetries: Int, + success: @escaping (String) -> Void, + failure: @escaping (Error) -> Void) { - let recordID = CKRecordID(recordName: recordName) + database().save(record) { + (_, error) in - let myContainer = CKContainer.default() - let privateDatabase = myContainer.privateCloudDatabase - privateDatabase.delete(withRecordID: recordID) { - (record, error) in - - if let error = error { - Logger.error("\(self.logTag) error deleting record: \(error)") - failure(error) - } else { - Logger.info("\(self.logTag) deleted record.") - success() + let outcome = outcomeForCloudKitError(error: error, + remainingRetries: remainingRetries, + label: "Save Record") + switch outcome { + case .success: + let recordName = record.recordID.recordName + success(recordName) + case .failureDoNotRetry(let outcomeError): + failure(outcomeError) + case .failureRetryAfterDelay(let retryDelay): + DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: { + saveRecordToCloud(record: record, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) + }) + case .failureRetryWithoutDelay: + DispatchQueue.global().async { + saveRecordToCloud(record: record, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) + } + case .unknownItem: + owsFail("\(self.logTag) unexpected CloudKit response.") + failure(invalidServiceResponseError()) } } } @@ -150,6 +163,7 @@ import CloudKit failure: @escaping (Error) -> Void) { checkForFileInCloud(recordName: recordName, + remainingRetries: maxRetries, success: { (record) in if let record = record { // Record found, updating existing record. @@ -183,6 +197,7 @@ import CloudKit failure: @escaping (Error) -> Void) { checkForFileInCloud(recordName: recordName, + remainingRetries: maxRetries, success: { (record) in if record != nil { // Record found, skipping save. @@ -207,42 +222,104 @@ import CloudKit failure: failure) } + // MARK: - Delete + + @objc + public class func deleteRecordFromCloud(recordName: String, + success: @escaping (()) -> Void, + failure: @escaping (Error) -> Void) { + deleteRecordFromCloud(recordName: recordName, + remainingRetries: maxRetries, + success: success, + failure: failure) + } + + private class func deleteRecordFromCloud(recordName: String, + remainingRetries: Int, + success: @escaping (()) -> Void, + failure: @escaping (Error) -> Void) { + + let recordID = CKRecordID(recordName: recordName) + + database().delete(withRecordID: recordID) { + (_, error) in + + let outcome = outcomeForCloudKitError(error: error, + remainingRetries: remainingRetries, + label: "Delete Record") + switch outcome { + case .success: + success() + case .failureDoNotRetry(let outcomeError): + failure(outcomeError) + case .failureRetryAfterDelay(let retryDelay): + DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: { + deleteRecordFromCloud(recordName: recordName, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) + }) + case .failureRetryWithoutDelay: + DispatchQueue.global().async { + deleteRecordFromCloud(recordName: recordName, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) + } + case .unknownItem: + owsFail("\(self.logTag) unexpected CloudKit response.") + failure(invalidServiceResponseError()) + } + } + } + + // MARK: - Exists? + private class func checkForFileInCloud(recordName: String, - success: @escaping (CKRecord?) -> Void, - failure: @escaping (Error) -> Void) { + remainingRetries: Int, + success: @escaping (CKRecord?) -> Void, + failure: @escaping (Error) -> Void) { let recordId = CKRecordID(recordName: recordName) let fetchOperation = CKFetchRecordsOperation(recordIDs: [recordId ]) // Don't download the file; we're just using the fetch to check whether or // not this record already exists. fetchOperation.desiredKeys = [] fetchOperation.perRecordCompletionBlock = { (record, recordId, error) in - if let error = error { - if let ckerror = error as? CKError { - if ckerror.code == .unknownItem { - // Record not found. - success(nil) - return - } - Logger.error("\(self.logTag) error fetching record: \(error) \(ckerror.code).") - } else { - Logger.error("\(self.logTag) error fetching record: \(error).") + + let outcome = outcomeForCloudKitError(error: error, + remainingRetries: remainingRetries, + label: "Check for Record") + switch outcome { + case .success: + guard let record = record else { + owsFail("\(self.logTag) missing fetching record.") + failure(invalidServiceResponseError()) + return } - failure(error) - return + // Record found. + success(record) + case .failureDoNotRetry(let outcomeError): + failure(outcomeError) + case .failureRetryAfterDelay(let retryDelay): + DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: { + checkForFileInCloud(recordName: recordName, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) + }) + case .failureRetryWithoutDelay: + DispatchQueue.global().async { + checkForFileInCloud(recordName: recordName, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) + } + case .unknownItem: + // Record not found. + success(nil) } - guard let record = record else { - Logger.error("\(self.logTag) missing fetching record.") - failure(OWSErrorWithCodeDescription(.exportBackupError, - NSLocalizedString("BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED", - comment: "Error indicating the a backup export failed to save a file to the cloud."))) - return - } - // Record found. - success(record) } - let myContainer = CKContainer.default() - let privateDatabase = myContainer.privateCloudDatabase - privateDatabase.add(fetchOperation) + database().add(fetchOperation) } @objc @@ -250,6 +327,7 @@ import CloudKit failure: @escaping (Error) -> Void) { checkForFileInCloud(recordName: manifestRecordName, + remainingRetries: maxRetries, success: { (record) in success(record != nil) }, @@ -265,6 +343,7 @@ import CloudKit fetchAllRecordNamesStep(query: query, previousRecordNames: [String](), cursor: nil, + remainingRetries: maxRetries, success: success, failure: failure) } @@ -272,12 +351,13 @@ import CloudKit private class func fetchAllRecordNamesStep(query: CKQuery, previousRecordNames: [String], cursor: CKQueryCursor?, + remainingRetries: Int, success: @escaping ([String]) -> Void, failure: @escaping (Error) -> Void) { var allRecordNames = previousRecordNames - let queryOperation = CKQueryOperation(query: query) + let queryOperation = CKQueryOperation(query: query) // If this isn't the first page of results for this query, resume // where we left off. queryOperation.cursor = cursor @@ -288,37 +368,62 @@ import CloudKit allRecordNames.append(record.recordID.recordName) } queryOperation.queryCompletionBlock = { (cursor, error) in - if let error = error { - Logger.error("\(self.logTag) error fetching all record names: \(error).") - failure(error) - return - } - if let cursor = cursor { - Logger.verbose("\(self.logTag) fetching more record names \(allRecordNames.count).") - // There are more pages of results, continue fetching. - fetchAllRecordNamesStep(query: query, - previousRecordNames: allRecordNames, - cursor: cursor, - success: success, - failure: failure) - return - } - Logger.info("\(self.logTag) fetched \(allRecordNames.count) record names.") - success(allRecordNames) - } - let myContainer = CKContainer.default() - let privateDatabase = myContainer.privateCloudDatabase - privateDatabase.add(queryOperation) + let outcome = outcomeForCloudKitError(error: error, + remainingRetries: remainingRetries, + label: "Fetch All Records") + switch outcome { + case .success: + if let cursor = cursor { + Logger.verbose("\(self.logTag) fetching more record names \(allRecordNames.count).") + // There are more pages of results, continue fetching. + fetchAllRecordNamesStep(query: query, + previousRecordNames: allRecordNames, + cursor: cursor, + remainingRetries: maxRetries, + success: success, + failure: failure) + return + } + Logger.info("\(self.logTag) fetched \(allRecordNames.count) record names.") + success(allRecordNames) + case .failureDoNotRetry(let outcomeError): + failure(outcomeError) + case .failureRetryAfterDelay(let retryDelay): + DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: { + fetchAllRecordNamesStep(query: query, + previousRecordNames: allRecordNames, + cursor: cursor, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) + }) + case .failureRetryWithoutDelay: + DispatchQueue.global().async { + fetchAllRecordNamesStep(query: query, + previousRecordNames: allRecordNames, + cursor: cursor, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) + } + case .unknownItem: + owsFail("\(self.logTag) unexpected CloudKit response.") + failure(invalidServiceResponseError()) + } + } + database().add(queryOperation) } + // MARK: - Download + @objc public class func downloadManifestFromCloud( - success: @escaping (Data) -> Void, - failure: @escaping (Error) -> Void) { + success: @escaping (Data) -> Void, + failure: @escaping (Error) -> Void) { downloadDataFromCloud(recordName: manifestRecordName, - success: success, - failure: failure) + success: success, + failure: failure) } @objc @@ -327,6 +432,7 @@ import CloudKit failure: @escaping (Error) -> Void) { downloadFromCloud(recordName: recordName, + remainingRetries: maxRetries, success: { (asset) in DispatchQueue.global().async { do { @@ -334,9 +440,7 @@ import CloudKit success(data) } catch { Logger.error("\(self.logTag) couldn't load asset file: \(error).") - failure(OWSErrorWithCodeDescription(.exportBackupError, - NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED", - comment: "Error indicating the a backup import failed to download a file from the cloud."))) + failure(invalidServiceResponseError()) } } }, @@ -350,6 +454,7 @@ import CloudKit failure: @escaping (Error) -> Void) { downloadFromCloud(recordName: recordName, + remainingRetries: maxRetries, success: { (asset) in DispatchQueue.global().async { do { @@ -357,48 +462,70 @@ import CloudKit success() } catch { Logger.error("\(self.logTag) couldn't copy asset file: \(error).") - failure(OWSErrorWithCodeDescription(.exportBackupError, - NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED", - comment: "Error indicating the a backup import failed to download a file from the cloud."))) + failure(invalidServiceResponseError()) } } }, failure: failure) } + // We return the CKAsset and not its fileUrl because + // CloudKit offers no guarantees around how long it'll + // keep around the underlying file. Presumably we can + // defer cleanup by maintaining a strong reference to + // the asset. private class func downloadFromCloud(recordName: String, - success: @escaping (CKAsset) -> Void, - failure: @escaping (Error) -> Void) { + remainingRetries: Int, + success: @escaping (CKAsset) -> Void, + failure: @escaping (Error) -> Void) { let recordId = CKRecordID(recordName: recordName) let fetchOperation = CKFetchRecordsOperation(recordIDs: [recordId ]) // Download all keys for this record. fetchOperation.perRecordCompletionBlock = { (record, recordId, error) in - if let error = error { - failure(error) - return - } - guard let record = record else { + + let outcome = outcomeForCloudKitError(error: error, + remainingRetries: remainingRetries, + label: "Download Record") + switch outcome { + case .success: + guard let record = record else { + Logger.error("\(self.logTag) missing fetching record.") + failure(invalidServiceResponseError()) + return + } + guard let asset = record[payloadKey] as? CKAsset else { + Logger.error("\(self.logTag) record missing payload.") + failure(invalidServiceResponseError()) + return + } + success(asset) + case .failureDoNotRetry(let outcomeError): + failure(outcomeError) + case .failureRetryAfterDelay(let retryDelay): + DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: { + downloadFromCloud(recordName: recordName, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) + }) + case .failureRetryWithoutDelay: + DispatchQueue.global().async { + downloadFromCloud(recordName: recordName, + remainingRetries: remainingRetries - 1, + success: success, + failure: failure) + } + case .unknownItem: Logger.error("\(self.logTag) missing fetching record.") - failure(OWSErrorWithCodeDescription(.exportBackupError, - NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED", - comment: "Error indicating the a backup import failed to download a file from the cloud."))) - return + failure(invalidServiceResponseError()) } - guard let asset = record[payloadKey] as? CKAsset else { - Logger.error("\(self.logTag) record missing payload.") - failure(OWSErrorWithCodeDescription(.exportBackupError, - NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED", - comment: "Error indicating the a backup import failed to download a file from the cloud."))) - return - } - success(asset) } - let myContainer = CKContainer.default() - let privateDatabase = myContainer.privateCloudDatabase - privateDatabase.add(fetchOperation) + database().add(fetchOperation) } + // MARK: - Access + @objc public class func checkCloudKitAccess(completion: @escaping (Bool) -> Void) { CKContainer.default().accountStatus(completionHandler: { (accountStatus, error) in @@ -422,4 +549,65 @@ import CloudKit } }) } + + // MARK: - Retry + + private enum APIOutcome { + case success + case failureDoNotRetry(error:Error) + case failureRetryAfterDelay(retryDelay: TimeInterval) + case failureRetryWithoutDelay + // This only applies to fetches. + case unknownItem + } + + private class func outcomeForCloudKitError(error: Error?, + remainingRetries: Int, + label: String) -> APIOutcome { + if let error = error as? CKError { + if error.code == CKError.unknownItem { + // This is not always an error for our purposes. + Logger.verbose("\(self.logTag) \(label) unknown item.") + return .unknownItem + } + + Logger.error("\(self.logTag) \(label) failed: \(error)") + + if remainingRetries < 1 { + Logger.verbose("\(self.logTag) \(label) no more retries.") + return .failureDoNotRetry(error:error) + } + + if #available(iOS 11, *) { + if error.code == CKError.serverResponseLost { + Logger.verbose("\(self.logTag) \(label) retry without delay.") + return .failureRetryWithoutDelay + } + } + + switch error { + case CKError.requestRateLimited, CKError.serviceUnavailable, CKError.zoneBusy: + let retryDelay = error.retryAfterSeconds ?? 3.0 + Logger.verbose("\(self.logTag) \(label) retry with delay: \(retryDelay).") + return .failureRetryAfterDelay(retryDelay:retryDelay) + case CKError.networkFailure: + Logger.verbose("\(self.logTag) \(label) retry without delay.") + return .failureRetryWithoutDelay + default: + Logger.verbose("\(self.logTag) \(label) unknown CKError.") + return .failureDoNotRetry(error:error) + } + } else if let error = error { + Logger.error("\(self.logTag) \(label) failed: \(error)") + if remainingRetries < 1 { + Logger.verbose("\(self.logTag) \(label) no more retries.") + return .failureDoNotRetry(error:error) + } + Logger.verbose("\(self.logTag) \(label) unknown error.") + return .failureDoNotRetry(error:error) + } else { + Logger.info("\(self.logTag) \(label) succeeded.") + return .success + } + } } diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index 1f7c2d67a6..3a831c4cb5 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -3,6 +3,7 @@ // #import "OWSBackupExportJob.h" +#import "OWSDatabaseMigration.h" #import "Signal-Swift.h" #import #import @@ -10,6 +11,7 @@ #import #import #import +#import #import #import #import @@ -84,10 +86,10 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe @interface OWSBackupExportJob () -@property (nonatomic, nullable) OWSBackupStorage *backupStorage; - @property (nonatomic, nullable) OWSBackgroundTask *backgroundTask; +@property (nonatomic, nullable) OWSBackupStorage *backupStorage; + @property (nonatomic) NSMutableArray *databaseFilePaths; // A map of "record name"-to-"file name". @property (nonatomic) NSMutableDictionary *databaseRecordMap; @@ -133,8 +135,8 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe progress:nil]; __weak OWSBackupExportJob *weakSelf = self; - [self configureExport:^(BOOL success) { - if (!success) { + [self configureExport:^(BOOL configureExportSuccess) { + if (!configureExportSuccess) { [self failWithErrorDescription: NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", @"Error indicating the a backup export could not export the user's data.")]; @@ -147,26 +149,29 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_EXPORT", @"Indicates that the backup export data is being exported.") progress:nil]; - if (![self exportDatabase]) { - [self failWithErrorDescription: - NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", - @"Error indicating the a backup export could not export the user's data.")]; - return; - } - if (self.isComplete) { - return; - } - [self saveToCloud:^(NSError *_Nullable saveError) { - if (saveError) { - [weakSelf failWithError:saveError]; + [self exportDatabase:^(BOOL exportDatabaseSuccess) { + if (!exportDatabaseSuccess) { + [self failWithErrorDescription: + NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", + @"Error indicating the a backup export could not export the user's data.")]; return; } - [self cleanUpCloud:^(NSError *_Nullable cleanUpError) { - if (cleanUpError) { - [weakSelf failWithError:cleanUpError]; + + if (self.isComplete) { + return; + } + [self saveToCloud:^(NSError *_Nullable saveError) { + if (saveError) { + [weakSelf failWithError:saveError]; return; } - [weakSelf succeed]; + [self cleanUpCloud:^(NSError *_Nullable cleanUpError) { + if (cleanUpError) { + [weakSelf failWithError:cleanUpError]; + return; + } + [weakSelf succeed]; + }]; }]; }]; }]; @@ -183,6 +188,35 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe return completion(NO); } + // We need to verify that we have a valid account. + // Otherwise, if we re-register on another device, we + // continue to backup on our old device, overwriting + // backups from the new device. + // + // We use an arbitrary request that requires authentication + // to verify our account state. + TSRequest *currentSignedPreKey = [OWSRequestFactory currentSignedPreKeyRequest]; + [[TSNetworkManager sharedManager] makeRequest:currentSignedPreKey + success:^(NSURLSessionDataTask *task, NSDictionary *responseObject) { + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + completion(YES); + }); + } + failure:^(NSURLSessionDataTask *task, NSError *error) { + // TODO: We may want to surface this in the UI. + DDLogError(@"%@ could not verify account status: %@.", self.logTag, error); + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + completion(NO); + }); + }]; +} + +- (void)exportDatabase:(OWSBackupJobBoolCompletion)completion +{ + OWSAssert(completion); + + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + if (![OWSBackupJob generateRandomDatabaseKeySpecWithKeychainKey:kOWSBackup_ExportDatabaseKeySpec]) { OWSProdLogAndFail(@"%@ Could not generate database key spec for export.", self.logTag); return completion(NO); @@ -203,6 +237,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe } return databaseKeySpec; }; + self.backupStorage = [[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock]; if (!self.backupStorage) { @@ -211,17 +246,18 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe } // TODO: Do we really need to run these registrations on the main thread? + __weak OWSBackupExportJob *weakSelf = self; dispatch_async(dispatch_get_main_queue(), ^{ - [self.backupStorage runSyncRegistrations]; - [self.backupStorage runAsyncRegistrationsWithCompletion:^{ + [weakSelf.backupStorage runSyncRegistrations]; + [weakSelf.backupStorage runAsyncRegistrationsWithCompletion:^{ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - completion(YES); + completion([weakSelf exportDatabaseContents]); }); }]; }); } -- (BOOL)exportDatabase +- (BOOL)exportDatabaseContents { DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); @@ -240,11 +276,16 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe __block unsigned long long copiedInteractions = 0; __block unsigned long long copiedEntities = 0; __block unsigned long long copiedAttachments = 0; + __block unsigned long long copiedMigrations = 0; self.attachmentFilePathMap = [NSMutableDictionary new]; [primaryDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) { [tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { + [dstTransaction setObject:@(YES) + forKey:kOWSBackup_Snapshot_ValidKey + inCollection:kOWSBackup_Snapshot_Collection]; + // Copy threads. [srcTransaction enumerateKeysAndObjectsInCollection:[TSThread collection] @@ -266,7 +307,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe // Copy attachments. [srcTransaction - enumerateKeysAndObjectsInCollection:[TSAttachmentStream collection] + enumerateKeysAndObjectsInCollection:[TSAttachment collection] usingBlock:^(NSString *key, id object, BOOL *stop) { if (self.isComplete) { *stop = YES; @@ -322,15 +363,38 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe copiedInteractions++; copiedEntities++; }]; + + // Copy migrations. + [srcTransaction + enumerateKeysAndObjectsInCollection:[OWSDatabaseMigration collection] + usingBlock:^(NSString *key, id object, BOOL *stop) { + if (self.isComplete) { + *stop = YES; + return; + } + if (![object isKindOfClass:[OWSDatabaseMigration class]]) { + OWSProdLogAndFail( + @"%@ unexpected class: %@", self.logTag, [object class]); + return; + } + OWSDatabaseMigration *migration = object; + [migration saveWithTransaction:dstTransaction]; + copiedMigrations++; + copiedEntities++; + }]; }]; }]; + if (self.isComplete) { + return NO; + } // TODO: Should we do a database checkpoint? DDLogInfo(@"%@ copiedThreads: %llu", self.logTag, copiedThreads); DDLogInfo(@"%@ copiedMessages: %llu", self.logTag, copiedInteractions); DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities); DDLogInfo(@"%@ copiedAttachments: %llu", self.logTag, copiedAttachments); + DDLogInfo(@"%@ copiedMigrations: %llu", self.logTag, copiedMigrations); [self.backupStorage logFileSizes]; @@ -460,6 +524,10 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe return; } strongSelf.attachmentRecordMap[recordName] = attachmentExport.relativeFilePath; + DDLogVerbose(@"%@ exported attachment: %@ as %@", + self.logTag, + attachmentFilePath, + attachmentExport.relativeFilePath); [strongSelf saveNextFileToCloud:completion]; }); } @@ -527,6 +595,8 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe kOWSBackup_ManifestKey_DatabaseKeySpec : databaseKeySpec.base64EncodedString, }; + DDLogVerbose(@"%@ json: %@", self.logTag, json); + NSError *error; NSData *_Nullable jsonData = [NSJSONSerialization dataWithJSONObject:json options:NSJSONWritingPrettyPrinted error:&error]; diff --git a/Signal/src/util/OWSBackupImportJob.m b/Signal/src/util/OWSBackupImportJob.m index c4c4310914..eae4ad2d90 100644 --- a/Signal/src/util/OWSBackupImportJob.m +++ b/Signal/src/util/OWSBackupImportJob.m @@ -3,6 +3,8 @@ // #import "OWSBackupImportJob.h" +#import "OWSDatabaseMigration.h" +#import "OWSDatabaseMigrationRunner.h" #import "Signal-Swift.h" #import #import @@ -22,6 +24,8 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe @property (nonatomic, nullable) OWSBackgroundTask *backgroundTask; +@property (nonatomic, nullable) OWSBackupStorage *backupStorage; + // A map of "record name"-to-"file name". @property (nonatomic) NSMutableDictionary *databaseRecordMap; @@ -124,7 +128,21 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return; } - [weakSelf succeed]; + [weakSelf ensureMigrations:^(BOOL ensureMigrationsSuccess) { + if (!ensureMigrationsSuccess) { + [weakSelf failWithErrorDescription:NSLocalizedString( + @"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT", + @"Error indicating the a backup import " + @"could not import the user's data.")]; + return; + } + + if (weakSelf.isComplete) { + return; + } + + [weakSelf succeed]; + }]; }]; }]; }]; @@ -187,6 +205,8 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return completion(NO); } + DDLogVerbose(@"%@ json: %@", self.logTag, json); + NSDictionary *_Nullable databaseRecordMap = json[kOWSBackup_ManifestKey_DatabaseFiles]; NSDictionary *_Nullable attachmentRecordMap = json[kOWSBackup_ManifestKey_AttachmentFiles]; NSString *_Nullable databaseKeySpecBase64 = json[kOWSBackup_ManifestKey_DatabaseKeySpec]; @@ -222,10 +242,11 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe // A map of "record name"-to-"downloaded file path". self.downloadedFileMap = [NSMutableDictionary new]; - [self downloadNextFileFromCloud:recordNames completion:completion]; + [self downloadNextFileFromCloud:recordNames recordCount:recordNames.count completion:completion]; } - (void)downloadNextFileFromCloud:(NSMutableArray *)recordNames + recordCount:(NSUInteger)recordCount completion:(OWSBackupJobCompletion)completion { OWSAssert(recordNames); @@ -243,11 +264,15 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe NSString *recordName = recordNames.lastObject; [recordNames removeLastObject]; + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_DOWNLOAD", + @"Indicates that the backup import data is being downloaded.") + progress:@((recordCount - recordNames.count) / (CGFloat)recordCount)]; + if (![recordName isKindOfClass:[NSString class]]) { DDLogError(@"%@ invalid record name in manifest: %@", self.logTag, [recordName class]); // Invalid record name in the manifest. This may be recoverable. // Ignore this for now and proceed with the other downloads. - return [self downloadNextFileFromCloud:recordNames completion:completion]; + return [self downloadNextFileFromCloud:recordNames recordCount:recordCount completion:completion]; } // Use a predictable file path so that multiple "import backup" attempts @@ -258,7 +283,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe if ([NSFileManager.defaultManager fileExistsAtPath:tempFilePath]) { [OWSFileSystem protectFileOrFolderAtPath:tempFilePath]; self.downloadedFileMap[recordName] = tempFilePath; - [self downloadNextFileFromCloud:recordNames completion:completion]; + [self downloadNextFileFromCloud:recordNames recordCount:recordCount completion:completion]; return; } @@ -268,7 +293,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ [OWSFileSystem protectFileOrFolderAtPath:tempFilePath]; self.downloadedFileMap[recordName] = tempFilePath; - [self downloadNextFileFromCloud:recordNames completion:completion]; + [self downloadNextFileFromCloud:recordNames recordCount:recordCount completion:completion]; }); } failure:^(NSError *error) { @@ -278,15 +303,22 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe - (void)restoreAttachmentFiles { - DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + DDLogVerbose(@"%@ %s: %zd", self.logTag, __PRETTY_FUNCTION__, self.attachmentRecordMap.count); NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder]; + NSUInteger count = 0; for (NSString *recordName in self.attachmentRecordMap) { if (self.isComplete) { return; } + count++; + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_FILES", + @"Indicates that the backup import data is being restored.") + progress:@(count / (CGFloat)self.attachmentRecordMap.count)]; + + NSString *dstRelativePath = self.attachmentRecordMap[recordName]; if (! [self restoreFileWithRecordName:recordName dstRelativePath:dstRelativePath dstDirPath:attachmentsDirPath]) { @@ -302,6 +334,10 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_DATABASE", + @"Indicates that the backup database is being restored.") + progress:nil]; + NSString *jobDatabaseDirPath = [self.jobTempDirPath stringByAppendingPathComponent:@"database"]; if (![OWSFileSystem ensureDirectoryExists:jobDatabaseDirPath]) { OWSProdLogAndFail(@"%@ Could not create jobDatabaseDirPath.", self.logTag); @@ -329,9 +365,9 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe } return databaseKeySpec; }; - OWSBackupStorage *_Nullable backupStorage = + self.backupStorage = [[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock]; - if (!backupStorage) { + if (!self.backupStorage) { OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag); return completion(NO); } @@ -339,19 +375,18 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe // TODO: Do we really need to run these registrations on the main thread? __weak OWSBackupImportJob *weakSelf = self; dispatch_async(dispatch_get_main_queue(), ^{ - [backupStorage runSyncRegistrations]; - [backupStorage runAsyncRegistrationsWithCompletion:^{ + [weakSelf.backupStorage runSyncRegistrations]; + [weakSelf.backupStorage runAsyncRegistrationsWithCompletion:^{ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - [weakSelf restoreDatabaseContents:backupStorage completion:completion]; - completion(YES); + [weakSelf restoreDatabaseContents:completion]; }); }]; }); } -- (void)restoreDatabaseContents:(OWSBackupStorage *)backupStorage completion:(OWSBackupJobBoolCompletion)completion +- (void)restoreDatabaseContents:(OWSBackupJobBoolCompletion)completion { - OWSAssert(backupStorage); + OWSAssert(self.backupStorage); OWSAssert(completion); DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); @@ -360,7 +395,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return completion(NO); } - YapDatabaseConnection *_Nullable tempDBConnection = backupStorage.newDatabaseConnection; + YapDatabaseConnection *_Nullable tempDBConnection = self.backupStorage.newDatabaseConnection; if (!tempDBConnection) { OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag); return completion(NO); @@ -371,97 +406,127 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return completion(NO); } - __block unsigned long long copiedThreads = 0; - __block unsigned long long copiedInteractions = 0; + NSDictionary *collectionTypeMap = @{ + [TSThread collection] : [TSThread class], + [TSAttachment collection] : [TSAttachment class], + [TSInteraction collection] : [TSInteraction class], + [OWSDatabaseMigration collection] : [OWSDatabaseMigration class], + }; + // Order matters here. + NSArray *collectionsToRestore = @[ + [TSThread collection], + [TSAttachment collection], + // Interactions refer to threads and attachments, + // so copy them afterward. + [TSInteraction collection], + [OWSDatabaseMigration collection], + ]; + NSMutableDictionary *restoredEntityCounts = [NSMutableDictionary new]; __block unsigned long long copiedEntities = 0; - __block unsigned long long copiedAttachments = 0; - + __block BOOL aborted = NO; [tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *srcTransaction) { [primaryDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { - // Copy threads. - [srcTransaction - enumerateKeysAndObjectsInCollection:[TSThread collection] - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - return; - } - if (![object isKindOfClass:[TSThread class]]) { - OWSProdLogAndFail( - @"%@ unexpected class: %@", self.logTag, [object class]); - return; - } - TSThread *thread = object; - [thread saveWithTransaction:dstTransaction]; - copiedThreads++; - copiedEntities++; - }]; + if (![srcTransaction boolForKey:kOWSBackup_Snapshot_ValidKey + inCollection:kOWSBackup_Snapshot_Collection + defaultValue:NO]) { + DDLogError(@"%@ invalid database.", self.logTag); + aborted = YES; + return completion(NO); + } - // Copy attachments. - [srcTransaction - enumerateKeysAndObjectsInCollection:[TSAttachmentStream collection] - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - return; - } - if (![object isKindOfClass:[TSAttachment class]]) { - OWSProdLogAndFail( - @"%@ unexpected class: %@", self.logTag, [object class]); - return; - } - TSAttachment *attachment = object; - [attachment saveWithTransaction:dstTransaction]; - copiedAttachments++; - copiedEntities++; - }]; + for (NSString *collection in collectionsToRestore) { + if ([collection isEqualToString:[OWSDatabaseMigration collection]]) { + // It's okay if there are existing migrations; we'll clear those + // before restoring. + continue; + } + if ([dstTransaction numberOfKeysInCollection:collection] > 0) { + DDLogError(@"%@ unexpected contents in database (%@).", self.logTag, collection); + } + } - // Copy interactions. + // Clear existing database contents. // - // Interactions refer to threads and attachments, so copy the last. - [srcTransaction - enumerateKeysAndObjectsInCollection:[TSInteraction collection] - usingBlock:^(NSString *key, id object, BOOL *stop) { - if (self.isComplete) { - *stop = YES; - return; - } - if (![object isKindOfClass:[TSInteraction class]]) { - OWSProdLogAndFail( - @"%@ unexpected class: %@", self.logTag, [object class]); - return; - } - // Ignore disappearing messages. - if ([object isKindOfClass:[TSMessage class]]) { - TSMessage *message = object; - if (message.isExpiringMessage) { - return; - } - } - TSInteraction *interaction = object; - // Ignore dynamic interactions. - if (interaction.isDynamicInteraction) { - return; - } - [interaction saveWithTransaction:dstTransaction]; - copiedInteractions++; - copiedEntities++; - }]; + // This should be safe since we only ever import into an empty database. + // + // Note that if the app receives a message after registering and before restoring + // backup, it will be lost. + // + // Note that this will clear all migrations. + for (NSString *collection in collectionsToRestore) { + [dstTransaction removeAllObjectsInCollection:collection]; + } + + // Copy database entities. + for (NSString *collection in collectionsToRestore) { + [srcTransaction enumerateKeysAndObjectsInCollection:collection + usingBlock:^(NSString *key, id object, BOOL *stop) { + if (self.isComplete) { + *stop = YES; + aborted = YES; + return; + } + Class expectedType = collectionTypeMap[collection]; + OWSAssert(expectedType); + if (![object isKindOfClass:expectedType]) { + OWSProdLogAndFail(@"%@ unexpected class: %@ != %@", + self.logTag, + [object class], + expectedType); + return; + } + TSYapDatabaseObject *databaseObject = object; + [databaseObject saveWithTransaction:dstTransaction]; + + NSUInteger count + = restoredEntityCounts[collection].unsignedIntValue; + restoredEntityCounts[collection] = @(count + 1); + copiedEntities++; + }]; + } }]; }]; - DDLogInfo(@"%@ copiedThreads: %llu", self.logTag, copiedThreads); - DDLogInfo(@"%@ copiedMessages: %llu", self.logTag, copiedInteractions); - DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities); - DDLogInfo(@"%@ copiedAttachments: %llu", self.logTag, copiedAttachments); + if (aborted) { + return; + } - [backupStorage logFileSizes]; + for (NSString *collection in collectionsToRestore) { + Class expectedType = collectionTypeMap[collection]; + OWSAssert(expectedType); + DDLogInfo(@"%@ copied %@ (%@): %@", self.logTag, expectedType, collection, restoredEntityCounts[collection]); + } + DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities); + + [self.backupStorage logFileSizes]; // Close the database. tempDBConnection = nil; - backupStorage = nil; + self.backupStorage = nil; - return completion(YES); + completion(YES); +} + +- (void)ensureMigrations:(OWSBackupJobBoolCompletion)completion +{ + OWSAssert(completion); + + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_FINALIZING", + @"Indicates that the backup import data is being finalized.") + progress:nil]; + + + // It's okay that we do this in a separate transaction from the + // restoration of backup contents. If some of migrations don't + // complete, they'll be run the next time the app launches. + dispatch_async(dispatch_get_main_queue(), ^{ + [[[OWSDatabaseMigrationRunner alloc] initWithPrimaryStorage:self.primaryStorage] + runAllOutstandingWithCompletion:^{ + completion(YES); + }]; + }); } - (BOOL)restoreFileWithRecordName:(NSString *)recordName @@ -483,8 +548,8 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe } NSString *dstFilePath = [dstDirPath stringByAppendingPathComponent:dstRelativePath]; if ([NSFileManager.defaultManager fileExistsAtPath:dstFilePath]) { - DDLogError(@"%@ skipping redundant file restore.", self.logTag); - return NO; + DDLogError(@"%@ skipping redundant file restore: %@.", self.logTag, dstFilePath); + return YES; } NSString *downloadedFilePath = self.downloadedFileMap[recordName]; if (![NSFileManager.defaultManager fileExistsAtPath:downloadedFilePath]) { @@ -498,6 +563,8 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return NO; } + DDLogError(@"%@ restored file: %@ (%@).", self.logTag, dstFilePath, dstRelativePath); + return YES; } diff --git a/Signal/src/util/OWSBackupJob.h b/Signal/src/util/OWSBackupJob.h index bb723a7f79..ee5b4aa0b8 100644 --- a/Signal/src/util/OWSBackupJob.h +++ b/Signal/src/util/OWSBackupJob.h @@ -8,6 +8,9 @@ extern NSString *const kOWSBackup_ManifestKey_DatabaseFiles; extern NSString *const kOWSBackup_ManifestKey_AttachmentFiles; extern NSString *const kOWSBackup_ManifestKey_DatabaseKeySpec; +extern NSString *const kOWSBackup_Snapshot_Collection; +extern NSString *const kOWSBackup_Snapshot_ValidKey; + typedef void (^OWSBackupJobBoolCompletion)(BOOL success); typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error); diff --git a/Signal/src/util/OWSBackupJob.m b/Signal/src/util/OWSBackupJob.m index 258e3db491..84ba42b56c 100644 --- a/Signal/src/util/OWSBackupJob.m +++ b/Signal/src/util/OWSBackupJob.m @@ -16,11 +16,15 @@ NSString *const kOWSBackup_ManifestKey_DatabaseKeySpec = @"database_key_spec"; NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; +NSString *const kOWSBackup_Snapshot_Collection = @"kOWSBackup_Snapshot_Collection"; +NSString *const kOWSBackup_Snapshot_ValidKey = @"kOWSBackup_Snapshot_ValidKey"; + @interface OWSBackupJob () @property (nonatomic, weak) id delegate; @property (atomic) BOOL isComplete; +@property (atomic) BOOL hasSucceeded; @property (nonatomic) OWSPrimaryStorage *primaryStorage; @@ -75,6 +79,10 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; OWSProdLogAndFail(@"%@ Could not create jobTempDirPath.", self.logTag); return NO; } + if (![OWSFileSystem protectFileOrFolderAtPath:self.jobTempDirPath]) { + OWSProdLogAndFail(@"%@ Could not protect jobTempDirPath.", self.logTag); + return NO; + } return YES; } @@ -93,9 +101,16 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; dispatch_async(dispatch_get_main_queue(), ^{ if (self.isComplete) { + OWSAssert(!self.hasSucceeded); return; } self.isComplete = YES; + + // There's a lot of asynchrony in these backup jobs; + // ensure we only end up finishing these jobs once. + OWSAssert(!self.hasSucceeded); + self.hasSucceeded = YES; + [self.delegate backupJobDidSucceed:self]; }); } @@ -110,6 +125,7 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; OWSProdLogAndFail(@"%@ %s %@", self.logTag, __PRETTY_FUNCTION__, error); dispatch_async(dispatch_get_main_queue(), ^{ + OWSAssert(!self.hasSucceeded); if (self.isComplete) { return; } diff --git a/Signal/translations/en.lproj/Localizable.strings b/Signal/translations/en.lproj/Localizable.strings index dbe987f29e..0278713a11 100644 --- a/Signal/translations/en.lproj/Localizable.strings +++ b/Signal/translations/en.lproj/Localizable.strings @@ -160,33 +160,45 @@ /* Error indicating the a backup export could not export the user's data. */ "BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT" = "Backup data could be exported."; +/* Error indicating that the app received an invalid response from CloudKit. */ +"BACKUP_EXPORT_ERROR_INVALID_CLOUDKIT_RESPONSE" = "Invalid server response."; + /* Error indicating the a backup export failed to save a file to the cloud. */ "BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED" = "Backup could not upload data."; /* Indicates that the cloud is being cleaned up. */ -"BACKUP_EXPORT_PHASE_CLEAN_UP" = "Cleaning up backup data."; +"BACKUP_EXPORT_PHASE_CLEAN_UP" = "Cleaning Up Backup"; /* Indicates that the backup export is being configured. */ -"BACKUP_EXPORT_PHASE_CONFIGURATION" = "Initializing backup."; +"BACKUP_EXPORT_PHASE_CONFIGURATION" = "Initializing Backup"; /* Indicates that the backup export data is being exported. */ -"BACKUP_EXPORT_PHASE_EXPORT" = "Exporting backup data."; +"BACKUP_EXPORT_PHASE_EXPORT" = "Exporting Backup"; /* Indicates that the backup export data is being uploaded. */ -"BACKUP_EXPORT_PHASE_UPLOAD" = "Uploading backup data."; +"BACKUP_EXPORT_PHASE_UPLOAD" = "Uploading Backup"; /* Error indicating the a backup import could not import the user's data. */ "BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT" = "Backup could not be imported."; -/* Error indicating the a backup import failed to download a file from the cloud. */ -"BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED" = "Could not download backup data."; - /* Indicates that the backup import is being configured. */ -"BACKUP_IMPORT_PHASE_CONFIGURATION" = "Configuring backup restore."; +"BACKUP_IMPORT_PHASE_CONFIGURATION" = "Configuring Backup"; + +/* Indicates that the backup import data is being downloaded. */ +"BACKUP_IMPORT_PHASE_DOWNLOAD" = "Downloading Backup Data"; + +/* Indicates that the backup import data is being finalized. */ +"BACKUP_IMPORT_PHASE_FINALIZING" = "Finalizing Backup"; /* Indicates that the backup import data is being imported. */ "BACKUP_IMPORT_PHASE_IMPORT" = "Importing backup."; +/* Indicates that the backup database is being restored. */ +"BACKUP_IMPORT_PHASE_RESTORING_DATABASE" = "Restoring Database"; + +/* Indicates that the backup import data is being restored. */ +"BACKUP_IMPORT_PHASE_RESTORING_FILES" = "Restoring Files"; + /* An explanation of the consequences of blocking another user. */ "BLOCK_BEHAVIOR_EXPLANATION" = "Blocked users will not be able to call you or send you messages."; @@ -1585,9 +1597,36 @@ /* Label for the backup view in app settings. */ "SETTINGS_BACKUP" = "Backup"; +/* Label for 'backup now' button in the backup settings view. */ +"SETTINGS_BACKUP_BACKUP_NOW" = "Backup Now"; + +/* Label for 'cancel backup' button in the backup settings view. */ +"SETTINGS_BACKUP_CANCEL_BACKUP" = "Cancel Backup"; + /* Label for switch in settings that controls whether or not backup is enabled. */ "SETTINGS_BACKUP_ENABLING_SWITCH" = "Backup Enabled"; +/* Label for phase row in the in the backup settings view. */ +"SETTINGS_BACKUP_PHASE" = "Phase"; + +/* Label for phase row in the in the backup settings view. */ +"SETTINGS_BACKUP_PROGRESS" = "Progress"; + +/* Label for status row in the in the backup settings view. */ +"SETTINGS_BACKUP_STATUS" = "Status"; + +/* Indicates that the last backup failed. */ +"SETTINGS_BACKUP_STATUS_FAILED" = "Backup Failed"; + +/* Indicates that app is not backing up. */ +"SETTINGS_BACKUP_STATUS_IDLE" = "Waiting"; + +/* Indicates that app is backing up. */ +"SETTINGS_BACKUP_STATUS_IN_PROGRESS" = "Backing Up"; + +/* Indicates that the last backup succeeded. */ +"SETTINGS_BACKUP_STATUS_SUCCEEDED" = "Backup Successful"; + /* A label for the 'add phone number' button in the block list table. */ "SETTINGS_BLOCK_LIST_ADD_BUTTON" = "Add…"; diff --git a/SignalMessaging/environment/migrations/OWS106EnsureProfileComplete.swift b/SignalMessaging/environment/migrations/OWS106EnsureProfileComplete.swift index dbeac5ae7e..d01092f125 100644 --- a/SignalMessaging/environment/migrations/OWS106EnsureProfileComplete.swift +++ b/SignalMessaging/environment/migrations/OWS106EnsureProfileComplete.swift @@ -21,11 +21,6 @@ public class OWS106EnsureProfileComplete: OWSDatabaseMigration { // Overriding runUp since we have some specific completion criteria which // is more likely to fail since it involves network requests. override public func runUp(completion:@escaping ((Void)) -> Void) { - guard type(of: self).sharedCompleteRegistrationFixerJob == nil else { - owsFail("\(self.TAG) should only be called once.") - return - } - let job = CompleteRegistrationFixerJob(completionHandler: { Logger.info("\(self.TAG) Completed. Saving.") self.save() diff --git a/SignalMessaging/environment/migrations/OWSDatabaseMigrationRunner.m b/SignalMessaging/environment/migrations/OWSDatabaseMigrationRunner.m index 1456c0d2e6..d5537cb1c8 100644 --- a/SignalMessaging/environment/migrations/OWSDatabaseMigrationRunner.m +++ b/SignalMessaging/environment/migrations/OWSDatabaseMigrationRunner.m @@ -56,52 +56,42 @@ NS_ASSUME_NONNULL_BEGIN - (void)runAllOutstandingWithCompletion:(OWSDatabaseMigrationCompletion)completion { - [self runMigrations:self.allMigrations completion:completion]; + [self runMigrations:[self.allMigrations mutableCopy] completion:completion]; } -- (void)runMigrations:(NSArray *)migrations +// Run migrations serially to: +// +// * Ensure predictable ordering. +// * Prevent them from interfering with each other (e.g. deadlock). +- (void)runMigrations:(NSMutableArray *)migrations completion:(OWSDatabaseMigrationCompletion)completion { OWSAssert(migrations); OWSAssert(completion); - NSMutableArray *migrationsToRun = [NSMutableArray new]; - for (OWSDatabaseMigration *migration in migrations) { - if ([OWSDatabaseMigration fetchObjectWithUniqueID:migration.uniqueId] == nil) { - [migrationsToRun addObject:migration]; - } - } - - if (migrationsToRun.count < 1) { + // If there are no more migrations to run, complete. + if (migrations.count < 1) { dispatch_async(dispatch_get_main_queue(), ^{ completion(); }); return; } - NSUInteger totalMigrationCount = migrationsToRun.count; - __block NSUInteger completedMigrationCount = 0; - // Call the completion exactly once, when the last migration completes. - void (^checkMigrationCompletion)(void) = ^{ - @synchronized(self) - { - completedMigrationCount++; - if (completedMigrationCount == totalMigrationCount) { - dispatch_async(dispatch_get_main_queue(), ^{ - completion(); - }); - } - } - }; + // Pop next migration from front of queue. + OWSDatabaseMigration *migration = migrations.firstObject; + [migrations removeObjectAtIndex:0]; - for (OWSDatabaseMigration *migration in migrationsToRun) { - if ([OWSDatabaseMigration fetchObjectWithUniqueID:migration.uniqueId]) { - DDLogDebug(@"%@ Skipping previously run migration: %@", self.logTag, migration); - } else { - DDLogWarn(@"%@ Running migration: %@", self.logTag, migration); - [migration runUpWithCompletion:checkMigrationCompletion]; - } + // If migration has already been run, skip it. + if ([OWSDatabaseMigration fetchObjectWithUniqueID:migration.uniqueId] != nil) { + [self runMigrations:migrations completion:completion]; + return; } + + DDLogInfo(@"%@ Running migration: %@", self.logTag, migration); + [migration runUpWithCompletion:^{ + DDLogInfo(@"%@ Migration complete: %@", self.logTag, migration); + [self runMigrations:migrations completion:completion]; + }]; } @end diff --git a/SignalServiceKit/src/Util/OWSError.h b/SignalServiceKit/src/Util/OWSError.h index 29c9bfa7c1..0a06d88caf 100644 --- a/SignalServiceKit/src/Util/OWSError.h +++ b/SignalServiceKit/src/Util/OWSError.h @@ -39,6 +39,8 @@ typedef NS_ENUM(NSInteger, OWSErrorCode) { OWSErrorCodeImportBackupFailed = 777417, // A possibly recoverable error occured while importing a backup. OWSErrorCodeImportBackupError = 777418, + // A non-recoverable while importing or exporting a backup. + OWSErrorCodeBackupFailure = 777419, }; extern NSString *const OWSErrorRecipientIdentifierKey;