// // Copyright 2021 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only // import Foundation import GRDB import LibSignalClient public class MessageProcessor { public static let messageProcessorDidDrainQueue = Notification.Name("messageProcessorDidDrainQueue") private var hasPendingEnvelopes: Bool { !pendingEnvelopes.isEmpty } public struct ProcessingTypes: OptionSet { public var rawValue: UInt8 public init(rawValue: UInt8) { self.rawValue = rawValue } public static let messageProcessor = ProcessingTypes(rawValue: 1 << 0) public static let groupMessageProcessor = ProcessingTypes(rawValue: 1 << 1) } public func waitForProcessingComplete(processingTypes: ProcessingTypes = [.messageProcessor, .groupMessageProcessor]) -> Guarantee { guard CurrentAppContext().shouldProcessIncomingMessages else { return Guarantee.value(()) } let shouldWaitForMessageProcessing: () -> Bool = { return processingTypes.contains(.messageProcessor) && self.hasPendingEnvelopes } if shouldWaitForMessageProcessing() { let messageProcessingPromise = NotificationCenter.default.observe(once: Self.messageProcessorDidDrainQueue) // We must check (again) after setting up the observer in case we miss the // notification. If you check before setting up the observer, the // notification might fire while the thread is sleeping. if shouldWaitForMessageProcessing() { return messageProcessingPromise.then { _ in // Recur, in case we've enqueued messages handled in another block. self.waitForProcessingComplete() }.asVoid() } } let shouldWaitForGroupMessageProcessing: () -> Bool = { return processingTypes.contains(.groupMessageProcessor) && SSKEnvironment.shared.databaseStorageRef.read { SSKEnvironment.shared.groupsV2MessageProcessorRef.hasPendingJobs(tx: $0) } } if shouldWaitForGroupMessageProcessing() { let groupMessageProcessingPromise = NotificationCenter.default.observe(once: GroupsV2MessageProcessor.didFlushGroupsV2MessageQueue) if shouldWaitForGroupMessageProcessing() { return groupMessageProcessingPromise.then { _ in // Recur, in case we've enqueued messages handled in another block. self.waitForProcessingComplete() }.asVoid() } } return Guarantee.value(()) } public func waitForFetchingAndProcessing() -> Guarantee { SSKEnvironment.shared.messageFetcherJobRef.waitForFetchingComplete().then { () -> Guarantee in self.waitForProcessingComplete() } } private let appReadiness: AppReadiness public init(appReadiness: AppReadiness) { self.appReadiness = appReadiness SwiftSingletons.register(self) NotificationCenter.default.addObserver( self, selector: #selector(registrationStateDidChange), name: .registrationStateDidChange, object: nil ) appReadiness.runNowOrWhenAppDidBecomeReadySync { SSKEnvironment.shared.messagePipelineSupervisorRef.register(pipelineStage: self) } } public func processReceivedEnvelopeData( _ envelopeData: Data, serverDeliveryTimestamp: UInt64, envelopeSource: EnvelopeSource, completion: @escaping (Error?) -> Void ) { guard !envelopeData.isEmpty else { completion(OWSAssertionError("Empty envelope, envelopeSource: \(envelopeSource).")) return } let protoEnvelope: SSKProtoEnvelope do { protoEnvelope = try SSKProtoEnvelope(serializedData: envelopeData) } catch { owsFailDebug("Failed to parse encrypted envelope \(error), envelopeSource: \(envelopeSource)") completion(error) return } // Drop any too-large messages on the floor. Well behaving clients should never send them. guard (protoEnvelope.content ?? Data()).count <= Self.maxEnvelopeByteCount else { completion(OWSAssertionError("Oversize envelope, envelopeSource: \(envelopeSource).")) return } processReceivedEnvelope( ReceivedEnvelope( envelope: protoEnvelope, serverDeliveryTimestamp: serverDeliveryTimestamp, completion: completion ), envelopeSource: envelopeSource ) } public func processReceivedEnvelope( _ envelopeProto: SSKProtoEnvelope, serverDeliveryTimestamp: UInt64, envelopeSource: EnvelopeSource, completion: @escaping (Error?) -> Void ) { processReceivedEnvelope( ReceivedEnvelope( envelope: envelopeProto, serverDeliveryTimestamp: serverDeliveryTimestamp, completion: completion ), envelopeSource: envelopeSource ) } private func processReceivedEnvelope(_ receivedEnvelope: ReceivedEnvelope, envelopeSource: EnvelopeSource) { let replacedEnvelope = pendingEnvelopes.enqueue(receivedEnvelope) if let replacedEnvelope { Logger.warn("Replaced \(replacedEnvelope.envelope.timestamp) serverGuid: \(replacedEnvelope.envelope.serverGuid as Optional)") replacedEnvelope.completion(MessageProcessingError.replacedEnvelope) } drainPendingEnvelopes() } private static let maxEnvelopeByteCount = 256 * 1024 private let serialQueue = DispatchQueue( label: "org.signal.message-processor", autoreleaseFrequency: .workItem ) private var pendingEnvelopes = PendingEnvelopes() private let isDrainingPendingEnvelopes = AtomicBool(false, lock: .init()) private func drainPendingEnvelopes() { guard CurrentAppContext().shouldProcessIncomingMessages else { return } guard DependenciesBridge.shared.tsAccountManager.registrationStateWithMaybeSneakyTransaction.isRegistered else { return } guard SSKEnvironment.shared.messagePipelineSupervisorRef.isMessageProcessingPermitted else { return } serialQueue.async { self.isDrainingPendingEnvelopes.set(true) while autoreleasepool(invoking: { self.drainNextBatch() }) {} self.isDrainingPendingEnvelopes.set(false) if self.pendingEnvelopes.isEmpty { NotificationCenter.default.postOnMainThread(name: Self.messageProcessorDidDrainQueue, object: nil) } } } /// Returns whether or not to continue draining the queue. private func drainNextBatch() -> Bool { assertOnQueue(serialQueue) guard SSKEnvironment.shared.messagePipelineSupervisorRef.isMessageProcessingPermitted else { return false } // We want a value that is just high enough to yield perf benefits. let kIncomingMessageBatchSize = 16 // If the app is in the background, use batch size of 1. // This reduces the risk of us never being able to drain any // messages from the queue. We should fine tune this number // to yield the best perf we can get. let batchSize = CurrentAppContext().isInBackground() ? 1 : kIncomingMessageBatchSize let batch = pendingEnvelopes.nextBatch(batchSize: batchSize) let batchEnvelopes = batch.batchEnvelopes let pendingEnvelopesCount = batch.pendingEnvelopesCount guard !batchEnvelopes.isEmpty else { return false } let startTime = CACurrentMediaTime() var processedEnvelopesCount = 0 SSKEnvironment.shared.databaseStorageRef.write { tx in // This is only called via `drainPendingEnvelopes`, and that confirms that // we're registered. If we're registered, we must have `LocalIdentifiers`, // so this (generally) shouldn't fail. guard let localIdentifiers = DependenciesBridge.shared.tsAccountManager.localIdentifiers(tx: tx) else { return } let localDeviceId = DependenciesBridge.shared.tsAccountManager.storedDeviceId(tx: tx) var remainingEnvelopes = batchEnvelopes[...] while !remainingEnvelopes.isEmpty { guard SSKEnvironment.shared.messagePipelineSupervisorRef.isMessageProcessingPermitted else { break } autoreleasepool { // If we build a request, we must handle it to ensure it's not lost if we // stop processing envelopes. let combinedRequest = buildNextCombinedRequest( envelopes: &remainingEnvelopes, localIdentifiers: localIdentifiers, localDeviceId: localDeviceId, tx: tx ) handle( combinedRequest: combinedRequest, localIdentifiers: localIdentifiers, transaction: tx ) } } processedEnvelopesCount += batchEnvelopes.count - remainingEnvelopes.count } pendingEnvelopes.removeProcessedEnvelopes(processedEnvelopesCount) let endTime = CACurrentMediaTime() let formattedDuration = String(format: "%.1f", (endTime - startTime) * 1000) Logger.info("Processed \(processedEnvelopesCount) envelopes (of \(pendingEnvelopesCount) total) in \(formattedDuration)ms") return true } // If envelopes is not empty, this will emit a single request for a non-delivery receipt or one or more requests // all for delivery receipts. private func buildNextCombinedRequest( envelopes: inout ArraySlice, localIdentifiers: LocalIdentifiers, localDeviceId: LocalDeviceId, tx: DBWriteTransaction ) -> RelatedProcessingRequests { let result = RelatedProcessingRequests() while let envelope = envelopes.first { envelopes.removeFirst() let request = processingRequest( for: envelope, localIdentifiers: localIdentifiers, localDeviceId: localDeviceId, tx: tx ) result.add(request) if request.deliveryReceiptMessageTimestamps == nil { // If we hit a non-delivery receipt envelope, handle it immediately to avoid // keeping potentially large decrypted envelopes in memory. break } } return result } private func handle(combinedRequest: RelatedProcessingRequests, localIdentifiers: LocalIdentifiers, transaction: DBWriteTransaction) { // Efficiently handle delivery receipts for the same message by fetching the sent message only // once and only using one updateWith... to update the message with new recipient state. BatchingDeliveryReceiptContext.withDeferredUpdates(transaction: transaction) { context in for request in combinedRequest.processingRequests { handleProcessingRequest(request, context: context, localIdentifiers: localIdentifiers, tx: transaction) } } } private func reallyHandleProcessingRequest( _ request: ProcessingRequest, context: DeliveryReceiptContext, localIdentifiers: LocalIdentifiers, transaction: DBWriteTransaction ) -> Error? { switch request.state { case .completed(error: let error): Logger.info("Envelope completed early with error \(String(describing: error))") return error case .enqueueForGroup(let decryptedEnvelope, let envelopeData): SSKEnvironment.shared.groupsV2MessageProcessorRef.enqueue( envelopeData: envelopeData, plaintextData: decryptedEnvelope.plaintextData, wasReceivedByUD: decryptedEnvelope.wasReceivedByUD, serverDeliveryTimestamp: request.receivedEnvelope.serverDeliveryTimestamp, tx: transaction ) SSKEnvironment.shared.messageReceiverRef.finishProcessingEnvelope(decryptedEnvelope, tx: transaction) return nil case .messageReceiverRequest(let messageReceiverRequest): SSKEnvironment.shared.messageReceiverRef.handleRequest(messageReceiverRequest, context: context, localIdentifiers: localIdentifiers, tx: transaction) SSKEnvironment.shared.messageReceiverRef.finishProcessingEnvelope(messageReceiverRequest.decryptedEnvelope, tx: transaction) return nil case .clearPlaceholdersOnly(let decryptedEnvelope): SSKEnvironment.shared.messageReceiverRef.finishProcessingEnvelope(decryptedEnvelope, tx: transaction) return nil case .serverReceipt(let serverReceiptEnvelope): SSKEnvironment.shared.messageReceiverRef.handleDeliveryReceipt(envelope: serverReceiptEnvelope, context: context, tx: transaction) return nil } } private func handleProcessingRequest( _ request: ProcessingRequest, context: DeliveryReceiptContext, localIdentifiers: LocalIdentifiers, tx: DBWriteTransaction ) { let error = reallyHandleProcessingRequest(request, context: context, localIdentifiers: localIdentifiers, transaction: tx) tx.addSyncCompletion { request.receivedEnvelope.completion(error) } } @objc private func registrationStateDidChange() { appReadiness.runNowOrWhenAppDidBecomeReadySync { self.drainPendingEnvelopes() } } public enum MessageAckBehavior { case shouldAck case shouldNotAck(error: Error) } public static func handleMessageProcessingOutcome(error: Error?) -> MessageAckBehavior { guard let error = error else { // Success. return .shouldAck } if case MessageProcessingError.replacedEnvelope = error { // _DO NOT_ ACK if de-duplicated before decryption. return .shouldNotAck(error: error) } else if case MessageProcessingError.blockedSender = error { return .shouldAck } else if let owsError = error as? OWSError, owsError.errorCode == OWSErrorCode.failedToDecryptDuplicateMessage.rawValue { // _DO_ ACK if de-duplicated during decryption. return .shouldAck } else { Logger.warn("Failed to process message: \(error)") // This should only happen for malformed envelopes. We may eventually // want to show an error in this case. return .shouldAck } } } private struct ProcessingRequest { enum State { case completed(error: Error?) case enqueueForGroup(decryptedEnvelope: DecryptedIncomingEnvelope, envelopeData: Data) case messageReceiverRequest(MessageReceiverRequest) case serverReceipt(ServerReceiptEnvelope) // Message decrypted but had an invalid protobuf. case clearPlaceholdersOnly(DecryptedIncomingEnvelope) } let receivedEnvelope: ReceivedEnvelope let state: State // If this request is for a delivery receipt, return the timestamps for the sent-messages it // corresponds to. var deliveryReceiptMessageTimestamps: [UInt64]? { switch state { case .completed, .enqueueForGroup, .clearPlaceholdersOnly: return nil case .serverReceipt(let envelope): return [envelope.validatedEnvelope.timestamp] case .messageReceiverRequest(let request): guard case .receiptMessage = request.messageType, let receiptMessage = request.protoContent.receiptMessage, receiptMessage.type == .delivery else { return nil } return receiptMessage.timestamp } } init(_ receivedEnvelope: ReceivedEnvelope, state: State) { self.receivedEnvelope = receivedEnvelope self.state = state } } private class RelatedProcessingRequests { private(set) var processingRequests = [ProcessingRequest]() func add(_ processingRequest: ProcessingRequest) { processingRequests.append(processingRequest) } } private struct ProcessingRequestBuilder { let receivedEnvelope: ReceivedEnvelope let blockingManager: BlockingManager let localDeviceId: LocalDeviceId let localIdentifiers: LocalIdentifiers let messageDecrypter: OWSMessageDecrypter let messageReceiver: MessageReceiver init( _ receivedEnvelope: ReceivedEnvelope, blockingManager: BlockingManager, localDeviceId: LocalDeviceId, localIdentifiers: LocalIdentifiers, messageDecrypter: OWSMessageDecrypter, messageReceiver: MessageReceiver ) { self.receivedEnvelope = receivedEnvelope self.blockingManager = blockingManager self.localDeviceId = localDeviceId self.localIdentifiers = localIdentifiers self.messageDecrypter = messageDecrypter self.messageReceiver = messageReceiver } func build(tx: DBWriteTransaction) -> ProcessingRequest.State { do { let decryptionResult = try receivedEnvelope.decryptIfNeeded( messageDecrypter: messageDecrypter, localIdentifiers: localIdentifiers, localDeviceId: localDeviceId, tx: tx ) switch decryptionResult { case .serverReceipt(let receiptEnvelope): return .serverReceipt(receiptEnvelope) case .decryptedMessage(let decryptedEnvelope): return processingRequest(for: decryptedEnvelope, tx: tx) } } catch { return .completed(error: error) } } private enum ProcessingStep { case discard case enqueueForGroupProcessing case processNow(shouldDiscardVisibleMessages: Bool) } private func processingStep( for decryptedEnvelope: DecryptedIncomingEnvelope, tx: DBWriteTransaction ) -> ProcessingStep { guard let contentProto = decryptedEnvelope.content, let groupContextV2 = GroupsV2MessageProcessor.groupContextV2(from: contentProto) else { // Non-v2-group messages can be processed immediately. return .processNow(shouldDiscardVisibleMessages: false) } guard GroupsV2MessageProcessor.canContextBeProcessedImmediately( groupContext: groupContextV2, tx: tx ) else { // Some v2 group messages required group state to be // updated before they can be processed. return .enqueueForGroupProcessing } let discardMode = GroupsMessageProcessor.discardMode( forMessageFrom: decryptedEnvelope.sourceAci, groupContext: groupContextV2, tx: tx ) switch discardMode { case .discard: // Some v2 group messages should be discarded and not processed. return .discard case .doNotDiscard: return .processNow(shouldDiscardVisibleMessages: false) case .discardVisibleMessages: // Some v2 group messages should be processed, but discarding any "visible" // messages, e.g. text messages or calls. return .processNow(shouldDiscardVisibleMessages: true) } } private func processingRequest( for decryptedEnvelope: DecryptedIncomingEnvelope, tx: DBWriteTransaction ) -> ProcessingRequest.State { owsPrecondition(CurrentAppContext().shouldProcessIncomingMessages) // Pre-processing has to happen during the same transaction that performed // decryption. messageReceiver.preprocessEnvelope(decryptedEnvelope, tx: tx) // If the sender is in the block list, we can skip scheduling any additional processing. let sourceAddress = SignalServiceAddress(decryptedEnvelope.sourceAci) if blockingManager.isAddressBlocked(sourceAddress, transaction: tx) { Logger.info("Skipping processing for blocked envelope from \(decryptedEnvelope.sourceAci)") return .completed(error: MessageProcessingError.blockedSender) } if decryptedEnvelope.localIdentity == .pni { let identityManager = DependenciesBridge.shared.identityManager identityManager.setShouldSharePhoneNumber(with: decryptedEnvelope.sourceAci, tx: tx) } switch processingStep(for: decryptedEnvelope, tx: tx) { case .discard: // Do nothing. return .completed(error: nil) case .enqueueForGroupProcessing: // If we can't process the message immediately, we enqueue it for // for processing in the same transaction within which it was decrypted // to prevent data loss. let envelopeData: Data do { envelopeData = try decryptedEnvelope.envelope.serializedData() } catch { owsFailDebug("failed to reserialize envelope: \(error)") return .completed(error: error) } return .enqueueForGroup(decryptedEnvelope: decryptedEnvelope, envelopeData: envelopeData) case .processNow(let shouldDiscardVisibleMessages): // Envelopes can be processed immediately if they're: // 1. Not a GV2 message. // 2. A GV2 message that doesn't require updating the group. // // The advantage to processing the message immediately is that we can full // process the message in the same transaction that we used to decrypt it. // This results in a significant perf benefit verse queueing the message // and waiting for that queue to open new transactions and process // messages. The downside is that if we *fail* to process this message // (e.g. the app crashed or was killed), we'll have to re-decrypt again // before we process. This is safe since the decrypt operation would also // be rolled back (since the transaction didn't commit) and should be rare. messageReceiver.checkForUnknownLinkedDevice(in: decryptedEnvelope, tx: tx) let buildResult = MessageReceiverRequest.buildRequest( for: decryptedEnvelope, serverDeliveryTimestamp: receivedEnvelope.serverDeliveryTimestamp, shouldDiscardVisibleMessages: shouldDiscardVisibleMessages, tx: tx ) switch buildResult { case .discard: return .completed(error: nil) case .noContent: return .clearPlaceholdersOnly(decryptedEnvelope) case .request(let messageReceiverRequest): return .messageReceiverRequest(messageReceiverRequest) } } } } private extension MessageProcessor { func processingRequest( for envelope: ReceivedEnvelope, localIdentifiers: LocalIdentifiers, localDeviceId: LocalDeviceId, tx: DBWriteTransaction ) -> ProcessingRequest { assertOnQueue(serialQueue) let builder = ProcessingRequestBuilder( envelope, blockingManager: SSKEnvironment.shared.blockingManagerRef, localDeviceId: localDeviceId, localIdentifiers: localIdentifiers, messageDecrypter: SSKEnvironment.shared.messageDecrypterRef, messageReceiver: SSKEnvironment.shared.messageReceiverRef ) return ProcessingRequest(envelope, state: builder.build(tx: tx)) } } // MARK: - extension MessageProcessor: MessageProcessingPipelineStage { public func supervisorDidResumeMessageProcessing(_ supervisor: MessagePipelineSupervisor) { drainPendingEnvelopes() } } // MARK: - private struct ReceivedEnvelope { let envelope: SSKProtoEnvelope let serverDeliveryTimestamp: UInt64 let completion: (Error?) -> Void enum DecryptionResult { case serverReceipt(ServerReceiptEnvelope) case decryptedMessage(DecryptedIncomingEnvelope) } func decryptIfNeeded( messageDecrypter: OWSMessageDecrypter, localIdentifiers: LocalIdentifiers, localDeviceId: LocalDeviceId, tx: DBWriteTransaction ) throws -> DecryptionResult { // Figure out what type of envelope we're dealing with. let validatedEnvelope = try ValidatedIncomingEnvelope(envelope, localIdentifiers: localIdentifiers) switch validatedEnvelope.kind { case .serverReceipt: return .serverReceipt(try ServerReceiptEnvelope(validatedEnvelope)) case .identifiedSender(let cipherType): return .decryptedMessage( try messageDecrypter.decryptIdentifiedEnvelope( validatedEnvelope, cipherType: cipherType, localIdentifiers: localIdentifiers, tx: tx ) ) case .unidentifiedSender: return .decryptedMessage( try messageDecrypter.decryptUnidentifiedSenderEnvelope( validatedEnvelope, localIdentifiers: localIdentifiers, localDeviceId: localDeviceId, tx: tx ) ) } } func isDuplicateOf(_ other: ReceivedEnvelope) -> Bool { guard let serverGuid = self.envelope.serverGuid else { owsFailDebug("Missing serverGuid.") return false } guard let otherServerGuid = other.envelope.serverGuid else { owsFailDebug("Missing other.serverGuid.") return false } return serverGuid == otherServerGuid } } // MARK: - public enum EnvelopeSource { case unknown case websocketIdentified case websocketUnidentified case rest // We re-decrypt incoming messages after accepting a safety number change. case identityChangeError case debugUI case tests } // MARK: - private class PendingEnvelopes { private let unfairLock = UnfairLock() private var pendingEnvelopes = [ReceivedEnvelope]() var isEmpty: Bool { unfairLock.withLock { pendingEnvelopes.isEmpty } } var count: Int { unfairLock.withLock { pendingEnvelopes.count } } struct Batch { let batchEnvelopes: [ReceivedEnvelope] let pendingEnvelopesCount: Int } func nextBatch(batchSize: Int) -> Batch { unfairLock.withLock { Batch( batchEnvelopes: Array(pendingEnvelopes.prefix(batchSize)), pendingEnvelopesCount: pendingEnvelopes.count ) } } func removeProcessedEnvelopes(_ processedEnvelopesCount: Int) { unfairLock.withLock { pendingEnvelopes.removeFirst(processedEnvelopesCount) } } func enqueue(_ receivedEnvelope: ReceivedEnvelope) -> ReceivedEnvelope? { return unfairLock.withLock { () -> ReceivedEnvelope? in if let indexToReplace = pendingEnvelopes.firstIndex(where: { receivedEnvelope.isDuplicateOf($0) }) { let replacedEnvelope = pendingEnvelopes[indexToReplace] pendingEnvelopes[indexToReplace] = receivedEnvelope return replacedEnvelope } else { pendingEnvelopes.append(receivedEnvelope) return nil } } } } // MARK: - public enum MessageProcessingError: Error { case wrongDestinationUuid case invalidMessageTypeForDestinationUuid case replacedEnvelope case blockedSender }