From e6178d54a6a64af6fa18d5c19bc0c4ed26bb59fc Mon Sep 17 00:00:00 2001 From: Nora Trapp Date: Fri, 13 Mar 2020 13:23:31 -0700 Subject: [PATCH] Use proto3 for storage service --- .../StorageServiceManager.swift | 70 ++++++------ .../StorageServiceProto+Sync.swift | 5 +- .../protobuf/StorageService.proto | 103 +++++++++--------- .../src/Util/StorageService.swift | 70 +++++++----- 4 files changed, 137 insertions(+), 111 deletions(-) diff --git a/SignalMessaging/Storage Service/StorageServiceManager.swift b/SignalMessaging/Storage Service/StorageServiceManager.swift index 70f4772ac7..9a726de9e0 100644 --- a/SignalMessaging/Storage Service/StorageServiceManager.swift +++ b/SignalMessaging/Storage Service/StorageServiceManager.swift @@ -503,7 +503,7 @@ class StorageServiceOperation: OWSOperation { } // Generate a fresh identifier - let storageIdentifier = StorageService.StorageIdentifier.generate() + let storageIdentifier = StorageService.StorageIdentifier.generate(type: .contact) accountIdentifierMap[accountId] = storageIdentifier let contactRecord = try StorageServiceProtoContactRecord.build(for: accountId, transaction: transaction) @@ -544,7 +544,7 @@ class StorageServiceOperation: OWSOperation { } // Generate a fresh identifier - let storageIdentifier = StorageService.StorageIdentifier.generate() + let storageIdentifier = StorageService.StorageIdentifier.generate(type: .groupv1) groupV1IdentifierMap[groupId] = storageIdentifier let groupV1Record = try StorageServiceProtoGroupV1Record.build(for: groupId, transaction: transaction) @@ -574,7 +574,7 @@ class StorageServiceOperation: OWSOperation { } // Generate a fresh identifier - let storageIdentifier = StorageService.StorageIdentifier.generate() + let storageIdentifier = StorageService.StorageIdentifier.generate(type: .groupv2) groupV2IdentifierMap[groupMasterKey] = storageIdentifier let groupV2Record = try StorageServiceProtoGroupV2Record.build(for: groupMasterKey, transaction: transaction) @@ -654,27 +654,28 @@ class StorageServiceOperation: OWSOperation { // Bump the manifest version version += 1 - let manifestBuilder = StorageServiceProtoManifestRecord.builder(version: version) - - let allKeys = (accountIdentifierMap.map { $1.data } + - groupV1IdentifierMap.map { $1.data } + - groupV2IdentifierMap.map { $1.data } + - unknownIdentifiers.map { $0.data }) - // We must persist any unknown identifiers, as they are potentially associated with // valid records that this version of the app doesn't yet understand how to parse. // Otherwise, this will cause ping-ponging with newer apps when they try and backup // new types of records, and then we subsequently delete them. - manifestBuilder.setKeys(allKeys) + let allIdentifiers = + accountIdentifierMap.map { $1 } + + groupV1IdentifierMap.map { $1 } + + groupV2IdentifierMap.map { $1 } + + unknownIdentifiers + + let manifestBuilder = StorageServiceProtoManifestRecord.builder(version: version) let manifest: StorageServiceProtoManifestRecord do { + manifestBuilder.setKeys(try allIdentifiers.map { try $0.buildRecord() }) + manifest = try manifestBuilder.build() } catch { return reportError(OWSAssertionError("failed to build proto with error: \(error)")) } - Logger.info("Backing up pending changes with manifest version: \(version). \(updatedItems.count) new items. \(deletedIdentifiers.count) deleted items. Total keys: \(allKeys.count)") + Logger.info("Backing up pending changes with manifest version: \(version). \(updatedItems.count) new items. \(deletedIdentifiers.count) deleted items. Total keys: \(allIdentifiers.count)") StorageService.updateManifest( manifest, @@ -770,7 +771,7 @@ class StorageServiceOperation: OWSOperation { databaseStorage.read { transaction in SignalRecipient.anyEnumerate(transaction: transaction) { recipient, _ in if recipient.devices.count > 0 { - let identifier = StorageService.StorageIdentifier.generate() + let identifier = StorageService.StorageIdentifier.generate(type: .contact) accountIdentifierMap[recipient.accountId] = identifier do { @@ -792,7 +793,7 @@ class StorageServiceOperation: OWSOperation { switch groupThread.groupModel.groupsVersion { case .V1: let groupId = groupThread.groupModel.groupId - let identifier = StorageService.StorageIdentifier.generate() + let identifier = StorageService.StorageIdentifier.generate(type: .groupv1) groupV1IdentifierMap[groupId] = identifier do { @@ -813,7 +814,7 @@ class StorageServiceOperation: OWSOperation { do { let groupMasterKey = try GroupsV2Protos.masterKeyData(forGroupModel: groupModel) - let identifier = StorageService.StorageIdentifier.generate() + let identifier = StorageService.StorageIdentifier.generate(type: .groupv2) groupV2IdentifierMap[groupMasterKey] = identifier let groupV2Record = try StorageServiceProtoGroupV2Record.build(for: groupMasterKey, transaction: transaction) @@ -830,10 +831,10 @@ class StorageServiceOperation: OWSOperation { } let manifestBuilder = StorageServiceProtoManifestRecord.builder(version: version) - manifestBuilder.setKeys(allItems.map { $0.identifier.data }) let manifest: StorageServiceProtoManifestRecord do { + manifestBuilder.setKeys(try allItems.map { try $0.identifier.buildRecord() }) manifest = try manifestBuilder.build() } catch { return reportError(OWSAssertionError("failed to build proto with error: \(error)")) @@ -883,7 +884,7 @@ class StorageServiceOperation: OWSOperation { var accountIdentifierMap: BidirectionalDictionary = [:] var groupV1IdentifierMap: BidirectionalDictionary = [:] var groupV2IdentifierMap: BidirectionalDictionary = [:] - var unknownIdentifiersTypeMap: [UInt32: [StorageService.StorageIdentifier]] = [:] + var unknownIdentifiersTypeMap: [StorageServiceProtoManifestRecordKeyType: [StorageService.StorageIdentifier]] = [:] var pendingAccountChanges: [AccountId: ChangeState] = [:] var pendingGroupV1Changes: [Data: ChangeState] = [:] var pendingGroupV2Changes: [Data: ChangeState] = [:] @@ -927,7 +928,7 @@ class StorageServiceOperation: OWSOperation { // of any items we don't know about locally. Since a new // id is always generated after a change, this should always // reflect the only items we need to fetch from the service. - let allManifestItems: Set = Set(manifest.keys.map { .init(data: $0) }) + let allManifestItems: Set = Set(manifest.keys.map { .init(data: $0.data, type: $0.type) }) // Cleanup our unknown identifiers type map to only reflect // identifiers that still exist in the manifest. @@ -1030,9 +1031,9 @@ class StorageServiceOperation: OWSOperation { // This is not a record type we know about yet, so record this identifier in // our unknown mapping. This allows us to skip fetching it in the future and // not accidentally blow it away when we push an update. - var unknownIdentifiersOfType = unknownIdentifiersTypeMap[item.record.type] ?? [] + var unknownIdentifiersOfType = unknownIdentifiersTypeMap[item.identifier.type] ?? [] unknownIdentifiersOfType.append(item.identifier) - unknownIdentifiersTypeMap[item.record.type] = unknownIdentifiersOfType + unknownIdentifiersTypeMap[item.identifier.type] = unknownIdentifiersOfType continue } @@ -1115,10 +1116,10 @@ class StorageServiceOperation: OWSOperation { databaseStorage.write { transaction in // We may have learned of new record types; if so we should // cull them from the unknownIdentifiersTypeMap on launch. - let knownTypes: [UInt32] = [ - UInt32(StorageServiceProtoStorageRecordType.contact.rawValue), - UInt32(StorageServiceProtoStorageRecordType.groupv1.rawValue), - UInt32(StorageServiceProtoStorageRecordType.groupv2.rawValue) + let knownTypes: [StorageServiceProtoManifestRecordKeyType] = [ + .contact, + .groupv1, + .groupv2 ] let oldUnknownIdentifiersTypeMap = StorageServiceOperation.unknownIdentifiersTypeMap(transaction: transaction) @@ -1160,7 +1161,7 @@ class StorageServiceOperation: OWSOperation { let dictionary = BidirectionalDictionary(anyDictionary) else { return [:] } - return dictionary.mapValues { .init(data: $0) } + return dictionary.mapValues { .init(data: $0, type: .contact) } } private static func setAccountToIdentifierMap( _ dictionary: BidirectionalDictionary, transaction: SDSAnyWriteTransaction) { @@ -1176,7 +1177,7 @@ class StorageServiceOperation: OWSOperation { let dictionary = BidirectionalDictionary(anyDictionary) else { return [:] } - return dictionary.mapValues { .init(data: $0) } + return dictionary.mapValues { .init(data: $0, type: .groupv1) } } private static func groupV2MasterKeyToIdentifierMap(transaction: SDSAnyReadTransaction) -> BidirectionalDictionary { @@ -1184,7 +1185,7 @@ class StorageServiceOperation: OWSOperation { let dictionary = BidirectionalDictionary(anyDictionary) else { return [:] } - return dictionary.mapValues { .init(data: $0) } + return dictionary.mapValues { .init(data: $0, type: .groupv2) } } private static func setGroupV1IdToIdentifierMap( _ dictionary: BidirectionalDictionary, transaction: SDSAnyWriteTransaction) { @@ -1207,14 +1208,19 @@ class StorageServiceOperation: OWSOperation { return unknownIdentifiersTypeMap(transaction: transaction).flatMap { $0.value } } - fileprivate static func unknownIdentifiersTypeMap(transaction: SDSAnyReadTransaction) -> [UInt32: [StorageService.StorageIdentifier]] { - guard let unknownIdentifiers = keyValueStore.getObject(unknownIdentifierTypeMapKey, transaction: transaction) as? [UInt32: [Data]] else { return [:] } - return unknownIdentifiers.mapValues { $0.map { .init(data: $0) } } + fileprivate static func unknownIdentifiersTypeMap(transaction: SDSAnyReadTransaction) -> [StorageServiceProtoManifestRecordKeyType: [StorageService.StorageIdentifier]] { + guard let unknownIdentifiers = keyValueStore.getObject(unknownIdentifierTypeMapKey, transaction: transaction) as? [Int: [Data]] else { return [:] } + return Dictionary(uniqueKeysWithValues: unknownIdentifiers.map { item in + let type: StorageServiceProtoManifestRecordKeyType = .UNRECOGNIZED(item.key) + return (type, item.value.map { .init(data: $0, type: type) }) + }) } - fileprivate static func setUnknownIdentifiersTypeMap( _ dictionary: [UInt32: [StorageService.StorageIdentifier]], transaction: SDSAnyWriteTransaction) { + fileprivate static func setUnknownIdentifiersTypeMap( _ dictionary: [StorageServiceProtoManifestRecordKeyType: [StorageService.StorageIdentifier]], transaction: SDSAnyWriteTransaction) { keyValueStore.setObject( - dictionary.mapValues { $0.map { $0.data }}, + Dictionary(uniqueKeysWithValues: dictionary.map { item in + return (item.key.rawValue, item.value.map { $0.data }) + }), key: unknownIdentifierTypeMapKey, transaction: transaction ) diff --git a/SignalMessaging/Storage Service/StorageServiceProto+Sync.swift b/SignalMessaging/Storage Service/StorageServiceProto+Sync.swift index c35529eda6..bdfc4f3cc0 100644 --- a/SignalMessaging/Storage Service/StorageServiceProto+Sync.swift +++ b/SignalMessaging/Storage Service/StorageServiceProto+Sync.swift @@ -234,6 +234,9 @@ extension StorageServiceProtoContactRecordIdentityState { return .default case .unverified: return .noLongerVerified + case .UNRECOGNIZED: + owsFailDebug("unrecognized verification state") + return .default } } } @@ -392,7 +395,7 @@ extension StorageServiceProtoGroupV2Record { case invalid // MARK: - CustomStringConvertible - + public var description: String { switch self { case .resolved: diff --git a/SignalServiceKit/protobuf/StorageService.proto b/SignalServiceKit/protobuf/StorageService.proto index 292e9017e8..d7cc638cc0 100644 --- a/SignalServiceKit/protobuf/StorageService.proto +++ b/SignalServiceKit/protobuf/StorageService.proto @@ -2,25 +2,16 @@ // Copyright (c) 2019 Open Whisper Systems. All rights reserved. // -// iOS - since we use a modern proto-compiler, we must specify -// the legacy proto format. -syntax = "proto2"; +syntax = "proto3"; // iOS - package name determines class prefix package StorageServiceProtos; message StorageItem { // @required - optional bytes key = 1; // Randomly generated 16-byte value + bytes key = 1; // Randomly generated 16-byte value // @required - optional bytes value = 2; // Encrypted serialized bytes -} - -message StorageManifest { - // @required - optional uint64 version = 1; // Monotonically increasing value - // @required - optional bytes value = 2; // Encrypted serialized bytes + bytes value = 2; // Encrypted serialized bytes } message StorageItems { @@ -28,30 +19,50 @@ message StorageItems { // keys that were found } +message StorageManifest { + // @required + uint64 version = 1; // Monotonically increasing value + // @required + bytes value = 2; // Encrypted ManifestRecord bytes +} + message ReadOperation { repeated bytes readKey = 1; } message WriteOperation { - optional StorageManifest manifest = 1; + StorageManifest manifest = 1; repeated StorageItem insertItem = 2; repeated bytes deleteKey = 3; - optional bool deleteAll = 4; + bool deleteAll = 4; } -message StorageRecord { - enum Type { - UNKNOWN = 0; - CONTACT = 1; - GROUPV1 = 2; - GROUPV2 = 3; +message ManifestRecord { + message Key { + enum Type { + UNKNOWN = 0; + CONTACT = 1; + GROUPV1 = 2; + GROUPV2 = 3; + } + + // @required + bytes data = 1; + // @required + Type type = 2; } // @required - optional uint32 type = 1; - optional ContactRecord contact = 2; - optional GroupV1Record groupV1 = 3; - optional GroupV2Record groupV2 = 4; + uint64 version = 1; + repeated Key keys = 2; +} + +message StorageRecord { + oneof record { + ContactRecord contact = 1; + GroupV1Record groupV1 = 2; + GroupV2Record groupV2 = 3; + } } message ContactRecord { @@ -62,42 +73,36 @@ message ContactRecord { UNVERIFIED = 2; } - optional bytes key = 1; - optional State state = 2; + bytes key = 1; + State state = 2; } message Profile { - optional string givenName = 1; - optional bytes key = 2; - optional string username = 3; - optional string familyName = 4; + string givenName = 1; + bytes key = 2; + string username = 3; + string familyName = 4; } - optional string serviceUuid = 1; - optional string serviceE164 = 2; - optional Profile profile = 3; - optional Identity identity = 4; - optional bool blocked = 5; - optional bool whitelisted = 6; - optional string nickname = 7; + string serviceUuid = 1; + string serviceE164 = 2; + Profile profile = 3; + Identity identity = 4; + bool blocked = 5; + bool whitelisted = 6; + string nickname = 7; } message GroupV1Record { // @required - optional bytes id = 1; - optional bool blocked = 2; - optional bool whitelisted = 3; + bytes id = 1; + bool blocked = 2; + bool whitelisted = 3; } message GroupV2Record { // @required - optional bytes masterKey = 1; - optional bool blocked = 2; - optional bool whitelisted = 3; -} - -message ManifestRecord { - // @required - optional uint64 version = 1; - repeated bytes keys = 2; + bytes masterKey = 1; + bool blocked = 2; + bool whitelisted = 3; } diff --git a/SignalServiceKit/src/Util/StorageService.swift b/SignalServiceKit/src/Util/StorageService.swift index 4852bb5cd1..70f22600be 100644 --- a/SignalServiceKit/src/Util/StorageService.swift +++ b/SignalServiceKit/src/Util/StorageService.swift @@ -73,14 +73,21 @@ public struct StorageService { public struct StorageIdentifier: Hashable { public static let identifierLength: Int32 = 16 public let data: Data + public let type: StorageServiceProtoManifestRecordKeyType - public init(data: Data) { + public init(data: Data, type: StorageServiceProtoManifestRecordKeyType) { if data.count != StorageIdentifier.identifierLength { owsFail("Initialized with invalid data") } self.data = data + self.type = type } - public static func generate() -> StorageIdentifier { - return .init(data: Randomness.generateRandomBytes(identifierLength)) + public static func generate(type: StorageServiceProtoManifestRecordKeyType) -> StorageIdentifier { + return .init(data: Randomness.generateRandomBytes(identifierLength), type: type) + } + + public func buildRecord() throws -> StorageServiceProtoManifestRecordKey { + let builder = StorageServiceProtoManifestRecordKey.builder(data: data, type: type) + return try builder.build() } } @@ -88,50 +95,50 @@ public struct StorageService { public let identifier: StorageIdentifier public let record: StorageServiceProtoStorageRecord - public var type: UInt32 { return record.type } + public var type: StorageServiceProtoManifestRecordKeyType { identifier.type } public var contactRecord: StorageServiceProtoContactRecord? { - guard type == StorageServiceProtoStorageRecordType.contact.rawValue else { return nil } - guard let contact = record.contact else { + guard case .contact = type else { return nil } + guard case .contact(let record) = record.record else { owsFailDebug("unexpectedly missing contact record") return nil } - return contact + return record } public var groupV1Record: StorageServiceProtoGroupV1Record? { - guard type == StorageServiceProtoStorageRecordType.groupv1.rawValue else { return nil } - guard let groupV1 = record.groupV1 else { + guard case .groupv1 = type else { return nil } + guard case .groupV1(let record) = record.record else { owsFailDebug("unexpectedly missing group v1 record") return nil } - return groupV1 + return record } public var groupV2Record: StorageServiceProtoGroupV2Record? { - guard type == StorageServiceProtoStorageRecordType.groupv2.rawValue else { return nil } - guard let groupV2 = record.groupV2 else { + guard case .groupv2 = type else { return nil } + guard case .groupV2(let record) = record.record else { owsFailDebug("unexpectedly missing group v2 record") return nil } - return groupV2 + return record } public init(identifier: StorageIdentifier, contact: StorageServiceProtoContactRecord) throws { - let storageRecord = StorageServiceProtoStorageRecord.builder(type: UInt32(StorageServiceProtoStorageRecordType.contact.rawValue)) - storageRecord.setContact(contact) + let storageRecord = StorageServiceProtoStorageRecord.builder() + storageRecord.setRecord(.contact(contact)) self.init(identifier: identifier, record: try storageRecord.build()) } public init(identifier: StorageIdentifier, groupV1: StorageServiceProtoGroupV1Record) throws { - let storageRecord = StorageServiceProtoStorageRecord.builder(type: UInt32(StorageServiceProtoStorageRecordType.groupv1.rawValue)) - storageRecord.setGroupV1(groupV1) + let storageRecord = StorageServiceProtoStorageRecord.builder() + storageRecord.setRecord(.groupV1(groupV1)) self.init(identifier: identifier, record: try storageRecord.build()) } public init(identifier: StorageIdentifier, groupV2: StorageServiceProtoGroupV2Record) throws { - let storageRecord = StorageServiceProtoStorageRecord.builder(type: UInt32(StorageServiceProtoStorageRecordType.groupv2.rawValue)) - storageRecord.setGroupV2(groupV2) + let storageRecord = StorageServiceProtoStorageRecord.builder() + storageRecord.setRecord(.groupV2(groupV2)) self.init(identifier: identifier, record: try storageRecord.build()) } @@ -288,9 +295,14 @@ public struct StorageService { let itemsProto = try StorageServiceProtoStorageItems.parseData(response.data) + let keyToIdentifier = Dictionary(uniqueKeysWithValues: keys.map { ($0.data, $0) }) + return try itemsProto.items.map { item in let encryptedItemData = item.value - let itemIdentifier = StorageIdentifier(data: item.key) + guard let itemIdentifier = keyToIdentifier[item.key] else { + owsFailDebug("missing identifier for fetched item") + throw StorageError.assertion + } let itemData: Data do { itemData = try KeyBackupService.decrypt( @@ -442,7 +454,7 @@ public extension StorageService { let testNames = ["abc", "def", "ghi", "jkl", "mno"] var recordsInManifest = [StorageItem]() for i in 0...4 { - let identifier = StorageService.StorageIdentifier.generate() + let identifier = StorageService.StorageIdentifier.generate(type: .contact) let contactRecordBuilder = StorageServiceProtoContactRecord.builder() contactRecordBuilder.setServiceUuid(testNames[i]) @@ -457,20 +469,20 @@ public extension StorageService { // Fetch Existing fetchLatestManifest().map { response in let previousVersion: UInt64 - var existingKeys: [Data]? + var existingKeys: [StorageIdentifier]? switch response { case .latestManifest(let latestManifest): previousVersion = latestManifest.version - existingKeys = latestManifest.keys + existingKeys = latestManifest.keys.map { StorageIdentifier(data: $0.data, type: $0.type ?? .unknown) } case .noNewerManifest, .noExistingManifest: previousVersion = ourManifestVersion } // set keys let newManifestBuilder = StorageServiceProtoManifestRecord.builder(version: ourManifestVersion) - newManifestBuilder.setKeys(recordsInManifest.map { $0.identifier.data }) + newManifestBuilder.setKeys(recordsInManifest.map { try! $0.identifier.buildRecord() }) - return (try! newManifestBuilder.build(), existingKeys?.map { StorageIdentifier(data: $0) } ?? []) + return (try! newManifestBuilder.build(), existingKeys ?? []) // Update or create initial manifest with test data }.then { latestManifest, deletedKeys in @@ -488,7 +500,7 @@ public extension StorageService { throw StorageError.assertion } - guard Set(latestManifest.keys) == Set(identifiersInManfest.map { $0.data }) else { + guard Set(latestManifest.keys.map { $0.data }) == Set(identifiersInManfest.map { $0.data }) else { owsFailDebug("manifest should only contain our test keys") throw StorageError.assertion } @@ -543,7 +555,7 @@ public extension StorageService { // Fetch a contact that doesn't exist }.then { - fetchItem(for: .generate()) + fetchItem(for: .generate(type: .contact)) }.map { item in guard item == nil else { owsFailDebug("this contact should not exist") @@ -584,12 +596,12 @@ public extension StorageService { }.map { let oldManifestBuilder = StorageServiceProtoManifestRecord.builder(version: 0) - let identifier = StorageIdentifier.generate() + let identifier = StorageIdentifier.generate(type: .contact) let recordBuilder = StorageServiceProtoContactRecord.builder() recordBuilder.setServiceUuid(testNames[0]) - oldManifestBuilder.setKeys([identifier.data]) + oldManifestBuilder.setKeys([try! identifier.buildRecord()]) return (try! oldManifestBuilder.build(), try! StorageItem(identifier: identifier, contact: try! recordBuilder.build())) }.then { oldManifest, item in