Use proto3 for storage service

This commit is contained in:
Nora Trapp 2020-03-13 13:23:31 -07:00
parent 45945a530c
commit e6178d54a6
4 changed files with 137 additions and 111 deletions

View File

@ -503,7 +503,7 @@ class StorageServiceOperation: OWSOperation {
}
// Generate a fresh identifier
let storageIdentifier = StorageService.StorageIdentifier.generate()
let storageIdentifier = StorageService.StorageIdentifier.generate(type: .contact)
accountIdentifierMap[accountId] = storageIdentifier
let contactRecord = try StorageServiceProtoContactRecord.build(for: accountId, transaction: transaction)
@ -544,7 +544,7 @@ class StorageServiceOperation: OWSOperation {
}
// Generate a fresh identifier
let storageIdentifier = StorageService.StorageIdentifier.generate()
let storageIdentifier = StorageService.StorageIdentifier.generate(type: .groupv1)
groupV1IdentifierMap[groupId] = storageIdentifier
let groupV1Record = try StorageServiceProtoGroupV1Record.build(for: groupId, transaction: transaction)
@ -574,7 +574,7 @@ class StorageServiceOperation: OWSOperation {
}
// Generate a fresh identifier
let storageIdentifier = StorageService.StorageIdentifier.generate()
let storageIdentifier = StorageService.StorageIdentifier.generate(type: .groupv2)
groupV2IdentifierMap[groupMasterKey] = storageIdentifier
let groupV2Record = try StorageServiceProtoGroupV2Record.build(for: groupMasterKey, transaction: transaction)
@ -654,27 +654,28 @@ class StorageServiceOperation: OWSOperation {
// Bump the manifest version
version += 1
let manifestBuilder = StorageServiceProtoManifestRecord.builder(version: version)
let allKeys = (accountIdentifierMap.map { $1.data } +
groupV1IdentifierMap.map { $1.data } +
groupV2IdentifierMap.map { $1.data } +
unknownIdentifiers.map { $0.data })
// We must persist any unknown identifiers, as they are potentially associated with
// valid records that this version of the app doesn't yet understand how to parse.
// Otherwise, this will cause ping-ponging with newer apps when they try and backup
// new types of records, and then we subsequently delete them.
manifestBuilder.setKeys(allKeys)
let allIdentifiers =
accountIdentifierMap.map { $1 } +
groupV1IdentifierMap.map { $1 } +
groupV2IdentifierMap.map { $1 } +
unknownIdentifiers
let manifestBuilder = StorageServiceProtoManifestRecord.builder(version: version)
let manifest: StorageServiceProtoManifestRecord
do {
manifestBuilder.setKeys(try allIdentifiers.map { try $0.buildRecord() })
manifest = try manifestBuilder.build()
} catch {
return reportError(OWSAssertionError("failed to build proto with error: \(error)"))
}
Logger.info("Backing up pending changes with manifest version: \(version). \(updatedItems.count) new items. \(deletedIdentifiers.count) deleted items. Total keys: \(allKeys.count)")
Logger.info("Backing up pending changes with manifest version: \(version). \(updatedItems.count) new items. \(deletedIdentifiers.count) deleted items. Total keys: \(allIdentifiers.count)")
StorageService.updateManifest(
manifest,
@ -770,7 +771,7 @@ class StorageServiceOperation: OWSOperation {
databaseStorage.read { transaction in
SignalRecipient.anyEnumerate(transaction: transaction) { recipient, _ in
if recipient.devices.count > 0 {
let identifier = StorageService.StorageIdentifier.generate()
let identifier = StorageService.StorageIdentifier.generate(type: .contact)
accountIdentifierMap[recipient.accountId] = identifier
do {
@ -792,7 +793,7 @@ class StorageServiceOperation: OWSOperation {
switch groupThread.groupModel.groupsVersion {
case .V1:
let groupId = groupThread.groupModel.groupId
let identifier = StorageService.StorageIdentifier.generate()
let identifier = StorageService.StorageIdentifier.generate(type: .groupv1)
groupV1IdentifierMap[groupId] = identifier
do {
@ -813,7 +814,7 @@ class StorageServiceOperation: OWSOperation {
do {
let groupMasterKey = try GroupsV2Protos.masterKeyData(forGroupModel: groupModel)
let identifier = StorageService.StorageIdentifier.generate()
let identifier = StorageService.StorageIdentifier.generate(type: .groupv2)
groupV2IdentifierMap[groupMasterKey] = identifier
let groupV2Record = try StorageServiceProtoGroupV2Record.build(for: groupMasterKey, transaction: transaction)
@ -830,10 +831,10 @@ class StorageServiceOperation: OWSOperation {
}
let manifestBuilder = StorageServiceProtoManifestRecord.builder(version: version)
manifestBuilder.setKeys(allItems.map { $0.identifier.data })
let manifest: StorageServiceProtoManifestRecord
do {
manifestBuilder.setKeys(try allItems.map { try $0.identifier.buildRecord() })
manifest = try manifestBuilder.build()
} catch {
return reportError(OWSAssertionError("failed to build proto with error: \(error)"))
@ -883,7 +884,7 @@ class StorageServiceOperation: OWSOperation {
var accountIdentifierMap: BidirectionalDictionary<AccountId, StorageService.StorageIdentifier> = [:]
var groupV1IdentifierMap: BidirectionalDictionary<Data, StorageService.StorageIdentifier> = [:]
var groupV2IdentifierMap: BidirectionalDictionary<Data, StorageService.StorageIdentifier> = [:]
var unknownIdentifiersTypeMap: [UInt32: [StorageService.StorageIdentifier]] = [:]
var unknownIdentifiersTypeMap: [StorageServiceProtoManifestRecordKeyType: [StorageService.StorageIdentifier]] = [:]
var pendingAccountChanges: [AccountId: ChangeState] = [:]
var pendingGroupV1Changes: [Data: ChangeState] = [:]
var pendingGroupV2Changes: [Data: ChangeState] = [:]
@ -927,7 +928,7 @@ class StorageServiceOperation: OWSOperation {
// of any items we don't know about locally. Since a new
// id is always generated after a change, this should always
// reflect the only items we need to fetch from the service.
let allManifestItems: Set<StorageService.StorageIdentifier> = Set(manifest.keys.map { .init(data: $0) })
let allManifestItems: Set<StorageService.StorageIdentifier> = Set(manifest.keys.map { .init(data: $0.data, type: $0.type) })
// Cleanup our unknown identifiers type map to only reflect
// identifiers that still exist in the manifest.
@ -1030,9 +1031,9 @@ class StorageServiceOperation: OWSOperation {
// This is not a record type we know about yet, so record this identifier in
// our unknown mapping. This allows us to skip fetching it in the future and
// not accidentally blow it away when we push an update.
var unknownIdentifiersOfType = unknownIdentifiersTypeMap[item.record.type] ?? []
var unknownIdentifiersOfType = unknownIdentifiersTypeMap[item.identifier.type] ?? []
unknownIdentifiersOfType.append(item.identifier)
unknownIdentifiersTypeMap[item.record.type] = unknownIdentifiersOfType
unknownIdentifiersTypeMap[item.identifier.type] = unknownIdentifiersOfType
continue
}
@ -1115,10 +1116,10 @@ class StorageServiceOperation: OWSOperation {
databaseStorage.write { transaction in
// We may have learned of new record types; if so we should
// cull them from the unknownIdentifiersTypeMap on launch.
let knownTypes: [UInt32] = [
UInt32(StorageServiceProtoStorageRecordType.contact.rawValue),
UInt32(StorageServiceProtoStorageRecordType.groupv1.rawValue),
UInt32(StorageServiceProtoStorageRecordType.groupv2.rawValue)
let knownTypes: [StorageServiceProtoManifestRecordKeyType] = [
.contact,
.groupv1,
.groupv2
]
let oldUnknownIdentifiersTypeMap = StorageServiceOperation.unknownIdentifiersTypeMap(transaction: transaction)
@ -1160,7 +1161,7 @@ class StorageServiceOperation: OWSOperation {
let dictionary = BidirectionalDictionary<AccountId, Data>(anyDictionary) else {
return [:]
}
return dictionary.mapValues { .init(data: $0) }
return dictionary.mapValues { .init(data: $0, type: .contact) }
}
private static func setAccountToIdentifierMap( _ dictionary: BidirectionalDictionary<AccountId, StorageService.StorageIdentifier>, transaction: SDSAnyWriteTransaction) {
@ -1176,7 +1177,7 @@ class StorageServiceOperation: OWSOperation {
let dictionary = BidirectionalDictionary<Data, Data>(anyDictionary) else {
return [:]
}
return dictionary.mapValues { .init(data: $0) }
return dictionary.mapValues { .init(data: $0, type: .groupv1) }
}
private static func groupV2MasterKeyToIdentifierMap(transaction: SDSAnyReadTransaction) -> BidirectionalDictionary<Data, StorageService.StorageIdentifier> {
@ -1184,7 +1185,7 @@ class StorageServiceOperation: OWSOperation {
let dictionary = BidirectionalDictionary<Data, Data>(anyDictionary) else {
return [:]
}
return dictionary.mapValues { .init(data: $0) }
return dictionary.mapValues { .init(data: $0, type: .groupv2) }
}
private static func setGroupV1IdToIdentifierMap( _ dictionary: BidirectionalDictionary<Data, StorageService.StorageIdentifier>, transaction: SDSAnyWriteTransaction) {
@ -1207,14 +1208,19 @@ class StorageServiceOperation: OWSOperation {
return unknownIdentifiersTypeMap(transaction: transaction).flatMap { $0.value }
}
fileprivate static func unknownIdentifiersTypeMap(transaction: SDSAnyReadTransaction) -> [UInt32: [StorageService.StorageIdentifier]] {
guard let unknownIdentifiers = keyValueStore.getObject(unknownIdentifierTypeMapKey, transaction: transaction) as? [UInt32: [Data]] else { return [:] }
return unknownIdentifiers.mapValues { $0.map { .init(data: $0) } }
fileprivate static func unknownIdentifiersTypeMap(transaction: SDSAnyReadTransaction) -> [StorageServiceProtoManifestRecordKeyType: [StorageService.StorageIdentifier]] {
guard let unknownIdentifiers = keyValueStore.getObject(unknownIdentifierTypeMapKey, transaction: transaction) as? [Int: [Data]] else { return [:] }
return Dictionary(uniqueKeysWithValues: unknownIdentifiers.map { item in
let type: StorageServiceProtoManifestRecordKeyType = .UNRECOGNIZED(item.key)
return (type, item.value.map { .init(data: $0, type: type) })
})
}
fileprivate static func setUnknownIdentifiersTypeMap( _ dictionary: [UInt32: [StorageService.StorageIdentifier]], transaction: SDSAnyWriteTransaction) {
fileprivate static func setUnknownIdentifiersTypeMap( _ dictionary: [StorageServiceProtoManifestRecordKeyType: [StorageService.StorageIdentifier]], transaction: SDSAnyWriteTransaction) {
keyValueStore.setObject(
dictionary.mapValues { $0.map { $0.data }},
Dictionary(uniqueKeysWithValues: dictionary.map { item in
return (item.key.rawValue, item.value.map { $0.data })
}),
key: unknownIdentifierTypeMapKey,
transaction: transaction
)

View File

@ -234,6 +234,9 @@ extension StorageServiceProtoContactRecordIdentityState {
return .default
case .unverified:
return .noLongerVerified
case .UNRECOGNIZED:
owsFailDebug("unrecognized verification state")
return .default
}
}
}
@ -392,7 +395,7 @@ extension StorageServiceProtoGroupV2Record {
case invalid
// MARK: - CustomStringConvertible
public var description: String {
switch self {
case .resolved:

View File

@ -2,25 +2,16 @@
// Copyright (c) 2019 Open Whisper Systems. All rights reserved.
//
// iOS - since we use a modern proto-compiler, we must specify
// the legacy proto format.
syntax = "proto2";
syntax = "proto3";
// iOS - package name determines class prefix
package StorageServiceProtos;
message StorageItem {
// @required
optional bytes key = 1; // Randomly generated 16-byte value
bytes key = 1; // Randomly generated 16-byte value
// @required
optional bytes value = 2; // Encrypted serialized bytes
}
message StorageManifest {
// @required
optional uint64 version = 1; // Monotonically increasing value
// @required
optional bytes value = 2; // Encrypted serialized bytes
bytes value = 2; // Encrypted serialized bytes
}
message StorageItems {
@ -28,30 +19,50 @@ message StorageItems {
// keys that were found
}
message StorageManifest {
// @required
uint64 version = 1; // Monotonically increasing value
// @required
bytes value = 2; // Encrypted ManifestRecord bytes
}
message ReadOperation {
repeated bytes readKey = 1;
}
message WriteOperation {
optional StorageManifest manifest = 1;
StorageManifest manifest = 1;
repeated StorageItem insertItem = 2;
repeated bytes deleteKey = 3;
optional bool deleteAll = 4;
bool deleteAll = 4;
}
message StorageRecord {
enum Type {
UNKNOWN = 0;
CONTACT = 1;
GROUPV1 = 2;
GROUPV2 = 3;
message ManifestRecord {
message Key {
enum Type {
UNKNOWN = 0;
CONTACT = 1;
GROUPV1 = 2;
GROUPV2 = 3;
}
// @required
bytes data = 1;
// @required
Type type = 2;
}
// @required
optional uint32 type = 1;
optional ContactRecord contact = 2;
optional GroupV1Record groupV1 = 3;
optional GroupV2Record groupV2 = 4;
uint64 version = 1;
repeated Key keys = 2;
}
message StorageRecord {
oneof record {
ContactRecord contact = 1;
GroupV1Record groupV1 = 2;
GroupV2Record groupV2 = 3;
}
}
message ContactRecord {
@ -62,42 +73,36 @@ message ContactRecord {
UNVERIFIED = 2;
}
optional bytes key = 1;
optional State state = 2;
bytes key = 1;
State state = 2;
}
message Profile {
optional string givenName = 1;
optional bytes key = 2;
optional string username = 3;
optional string familyName = 4;
string givenName = 1;
bytes key = 2;
string username = 3;
string familyName = 4;
}
optional string serviceUuid = 1;
optional string serviceE164 = 2;
optional Profile profile = 3;
optional Identity identity = 4;
optional bool blocked = 5;
optional bool whitelisted = 6;
optional string nickname = 7;
string serviceUuid = 1;
string serviceE164 = 2;
Profile profile = 3;
Identity identity = 4;
bool blocked = 5;
bool whitelisted = 6;
string nickname = 7;
}
message GroupV1Record {
// @required
optional bytes id = 1;
optional bool blocked = 2;
optional bool whitelisted = 3;
bytes id = 1;
bool blocked = 2;
bool whitelisted = 3;
}
message GroupV2Record {
// @required
optional bytes masterKey = 1;
optional bool blocked = 2;
optional bool whitelisted = 3;
}
message ManifestRecord {
// @required
optional uint64 version = 1;
repeated bytes keys = 2;
bytes masterKey = 1;
bool blocked = 2;
bool whitelisted = 3;
}

View File

@ -73,14 +73,21 @@ public struct StorageService {
public struct StorageIdentifier: Hashable {
public static let identifierLength: Int32 = 16
public let data: Data
public let type: StorageServiceProtoManifestRecordKeyType
public init(data: Data) {
public init(data: Data, type: StorageServiceProtoManifestRecordKeyType) {
if data.count != StorageIdentifier.identifierLength { owsFail("Initialized with invalid data") }
self.data = data
self.type = type
}
public static func generate() -> StorageIdentifier {
return .init(data: Randomness.generateRandomBytes(identifierLength))
public static func generate(type: StorageServiceProtoManifestRecordKeyType) -> StorageIdentifier {
return .init(data: Randomness.generateRandomBytes(identifierLength), type: type)
}
public func buildRecord() throws -> StorageServiceProtoManifestRecordKey {
let builder = StorageServiceProtoManifestRecordKey.builder(data: data, type: type)
return try builder.build()
}
}
@ -88,50 +95,50 @@ public struct StorageService {
public let identifier: StorageIdentifier
public let record: StorageServiceProtoStorageRecord
public var type: UInt32 { return record.type }
public var type: StorageServiceProtoManifestRecordKeyType { identifier.type }
public var contactRecord: StorageServiceProtoContactRecord? {
guard type == StorageServiceProtoStorageRecordType.contact.rawValue else { return nil }
guard let contact = record.contact else {
guard case .contact = type else { return nil }
guard case .contact(let record) = record.record else {
owsFailDebug("unexpectedly missing contact record")
return nil
}
return contact
return record
}
public var groupV1Record: StorageServiceProtoGroupV1Record? {
guard type == StorageServiceProtoStorageRecordType.groupv1.rawValue else { return nil }
guard let groupV1 = record.groupV1 else {
guard case .groupv1 = type else { return nil }
guard case .groupV1(let record) = record.record else {
owsFailDebug("unexpectedly missing group v1 record")
return nil
}
return groupV1
return record
}
public var groupV2Record: StorageServiceProtoGroupV2Record? {
guard type == StorageServiceProtoStorageRecordType.groupv2.rawValue else { return nil }
guard let groupV2 = record.groupV2 else {
guard case .groupv2 = type else { return nil }
guard case .groupV2(let record) = record.record else {
owsFailDebug("unexpectedly missing group v2 record")
return nil
}
return groupV2
return record
}
public init(identifier: StorageIdentifier, contact: StorageServiceProtoContactRecord) throws {
let storageRecord = StorageServiceProtoStorageRecord.builder(type: UInt32(StorageServiceProtoStorageRecordType.contact.rawValue))
storageRecord.setContact(contact)
let storageRecord = StorageServiceProtoStorageRecord.builder()
storageRecord.setRecord(.contact(contact))
self.init(identifier: identifier, record: try storageRecord.build())
}
public init(identifier: StorageIdentifier, groupV1: StorageServiceProtoGroupV1Record) throws {
let storageRecord = StorageServiceProtoStorageRecord.builder(type: UInt32(StorageServiceProtoStorageRecordType.groupv1.rawValue))
storageRecord.setGroupV1(groupV1)
let storageRecord = StorageServiceProtoStorageRecord.builder()
storageRecord.setRecord(.groupV1(groupV1))
self.init(identifier: identifier, record: try storageRecord.build())
}
public init(identifier: StorageIdentifier, groupV2: StorageServiceProtoGroupV2Record) throws {
let storageRecord = StorageServiceProtoStorageRecord.builder(type: UInt32(StorageServiceProtoStorageRecordType.groupv2.rawValue))
storageRecord.setGroupV2(groupV2)
let storageRecord = StorageServiceProtoStorageRecord.builder()
storageRecord.setRecord(.groupV2(groupV2))
self.init(identifier: identifier, record: try storageRecord.build())
}
@ -288,9 +295,14 @@ public struct StorageService {
let itemsProto = try StorageServiceProtoStorageItems.parseData(response.data)
let keyToIdentifier = Dictionary(uniqueKeysWithValues: keys.map { ($0.data, $0) })
return try itemsProto.items.map { item in
let encryptedItemData = item.value
let itemIdentifier = StorageIdentifier(data: item.key)
guard let itemIdentifier = keyToIdentifier[item.key] else {
owsFailDebug("missing identifier for fetched item")
throw StorageError.assertion
}
let itemData: Data
do {
itemData = try KeyBackupService.decrypt(
@ -442,7 +454,7 @@ public extension StorageService {
let testNames = ["abc", "def", "ghi", "jkl", "mno"]
var recordsInManifest = [StorageItem]()
for i in 0...4 {
let identifier = StorageService.StorageIdentifier.generate()
let identifier = StorageService.StorageIdentifier.generate(type: .contact)
let contactRecordBuilder = StorageServiceProtoContactRecord.builder()
contactRecordBuilder.setServiceUuid(testNames[i])
@ -457,20 +469,20 @@ public extension StorageService {
// Fetch Existing
fetchLatestManifest().map { response in
let previousVersion: UInt64
var existingKeys: [Data]?
var existingKeys: [StorageIdentifier]?
switch response {
case .latestManifest(let latestManifest):
previousVersion = latestManifest.version
existingKeys = latestManifest.keys
existingKeys = latestManifest.keys.map { StorageIdentifier(data: $0.data, type: $0.type ?? .unknown) }
case .noNewerManifest, .noExistingManifest:
previousVersion = ourManifestVersion
}
// set keys
let newManifestBuilder = StorageServiceProtoManifestRecord.builder(version: ourManifestVersion)
newManifestBuilder.setKeys(recordsInManifest.map { $0.identifier.data })
newManifestBuilder.setKeys(recordsInManifest.map { try! $0.identifier.buildRecord() })
return (try! newManifestBuilder.build(), existingKeys?.map { StorageIdentifier(data: $0) } ?? [])
return (try! newManifestBuilder.build(), existingKeys ?? [])
// Update or create initial manifest with test data
}.then { latestManifest, deletedKeys in
@ -488,7 +500,7 @@ public extension StorageService {
throw StorageError.assertion
}
guard Set(latestManifest.keys) == Set(identifiersInManfest.map { $0.data }) else {
guard Set(latestManifest.keys.map { $0.data }) == Set(identifiersInManfest.map { $0.data }) else {
owsFailDebug("manifest should only contain our test keys")
throw StorageError.assertion
}
@ -543,7 +555,7 @@ public extension StorageService {
// Fetch a contact that doesn't exist
}.then {
fetchItem(for: .generate())
fetchItem(for: .generate(type: .contact))
}.map { item in
guard item == nil else {
owsFailDebug("this contact should not exist")
@ -584,12 +596,12 @@ public extension StorageService {
}.map {
let oldManifestBuilder = StorageServiceProtoManifestRecord.builder(version: 0)
let identifier = StorageIdentifier.generate()
let identifier = StorageIdentifier.generate(type: .contact)
let recordBuilder = StorageServiceProtoContactRecord.builder()
recordBuilder.setServiceUuid(testNames[0])
oldManifestBuilder.setKeys([identifier.data])
oldManifestBuilder.setKeys([try! identifier.buildRecord()])
return (try! oldManifestBuilder.build(), try! StorageItem(identifier: identifier, contact: try! recordBuilder.build()))
}.then { oldManifest, item in