1633 lines
67 KiB
Swift
1633 lines
67 KiB
Swift
//
|
||
// Copyright 2025 Signal Messenger, LLC
|
||
// SPDX-License-Identifier: AGPL-3.0-only
|
||
//
|
||
|
||
import GRDB
|
||
import LibSignalClient
|
||
|
||
public struct NeedsListMediaError: Error {}
|
||
|
||
public struct ListMediaIntegrityCheckResult: Codable {
|
||
public struct Result: Codable {
|
||
/// Count of attachments we expected to see on CDN and did see on CDN.
|
||
/// This count is "good".
|
||
public fileprivate(set) var uploadedCount: Int
|
||
/// Count of attachments we did not expect to see on CDN (because they are ineligible
|
||
/// for backups, e.g. have a DM timer) and did not see on CDN.
|
||
/// This count is "good".
|
||
public fileprivate(set) var ineligibleCount: Int
|
||
/// Count of attachments we expected to see on CDN but did not.
|
||
/// This count is "bad".
|
||
public fileprivate(set) var missingFromCdnCount: Int
|
||
public fileprivate(set) var missingFromCdnSampleAttachmentIds: Set<Attachment.IDType>? = Set()
|
||
/// Count of attachments that exist locally, are eligible for upload, are not marked
|
||
/// uploaded, are not on the CDN, and therefore _should_ be in the upload
|
||
/// queue but are not in the upload queue.
|
||
/// This count is "bad".
|
||
public fileprivate(set) var notScheduledForUploadCount: Int? = 0
|
||
public fileprivate(set) var notScheduledForUploadSampleAttachmentIds: Set<Attachment.IDType>? = Set()
|
||
/// Count of attachments we did not expect to see on CDN but did see.
|
||
/// This count can be "bad" because it could indicate a bug with local state management,
|
||
/// but it could happen in normal edge cases if we just didn't know about a completed upload.
|
||
public fileprivate(set) var discoveredOnCdnCount: Int
|
||
public fileprivate(set) var discoveredOnCdnSampleAttachmentIds: Set<Attachment.IDType>? = Set()
|
||
|
||
static var empty: Result {
|
||
return Result(uploadedCount: 0, ineligibleCount: 0, missingFromCdnCount: 0, discoveredOnCdnCount: 0)
|
||
}
|
||
|
||
var hasFailures: Bool {
|
||
return missingFromCdnCount > 0 || (notScheduledForUploadCount ?? 0) > 0 || discoveredOnCdnCount > 0
|
||
}
|
||
|
||
mutating func addSampleId(_ id: Attachment.IDType, _ keyPath: WritableKeyPath<Result, Set<Attachment.IDType>?>) {
|
||
var sampleIds = self[keyPath: keyPath] ?? Set()
|
||
if sampleIds.count >= 10 {
|
||
// Only keep 10 ids
|
||
return
|
||
}
|
||
sampleIds.insert(id)
|
||
self[keyPath: keyPath] = sampleIds
|
||
}
|
||
}
|
||
|
||
public let listMediaStartTimestamp: UInt64
|
||
public fileprivate(set) var fullsize: Result
|
||
public fileprivate(set) var thumbnail: Result
|
||
/// Objects we discovered on CDN that don't match any local attachment;
|
||
/// we can't know if these were thumbnails or fullsize.
|
||
/// This count is "bad".
|
||
public fileprivate(set) var orphanedObjectCount: Int
|
||
|
||
var hasFailures: Bool {
|
||
if fullsize.uploadedCount == 0 {
|
||
// The first time we run list media, we have no
|
||
// uploads, so don't count as a failure.
|
||
return false
|
||
}
|
||
|
||
// Don't count thumbnail failures
|
||
// Don't count orphans; we maybe just haven't deleted yet.
|
||
return fullsize.hasFailures
|
||
}
|
||
}
|
||
|
||
public protocol BackupListMediaManager {
|
||
|
||
/// Returns true if a list media should be run whenever is next possible.
|
||
func getNeedsQueryListMedia(tx: DBReadTransaction) -> Bool
|
||
|
||
/// Wipes any persisted state related to list-media, such as artifacts of an
|
||
/// interrupted attempt or details about prior attempts.
|
||
func wipe(tx: DBWriteTransaction)
|
||
|
||
/// Perform a list-media operation, if necessary. This is a durable,
|
||
/// resumable, multi-stage operation.
|
||
func queryListMediaIfNeeded() async throws
|
||
}
|
||
|
||
class BackupListMediaManagerImpl: BackupListMediaManager {
|
||
|
||
private let accountKeyStore: AccountKeyStore
|
||
private let attachmentStore: AttachmentStore
|
||
private let attachmentUploadStore: AttachmentUploadStore
|
||
private let backupAttachmentDownloadProgress: BackupAttachmentDownloadProgress
|
||
private let backupAttachmentDownloadStore: BackupAttachmentDownloadStore
|
||
private let backupAttachmentUploadProgress: BackupAttachmentUploadProgress
|
||
private let backupAttachmentUploadScheduler: BackupAttachmentUploadScheduler
|
||
private let backupAttachmentUploadStore: BackupAttachmentUploadStore
|
||
private let backupAttachmentUploadEraStore: BackupAttachmentUploadEraStore
|
||
private let backupListMediaStore: BackupListMediaStore
|
||
private let backupMediaErrorNotificationPresenter: BackupMediaErrorNotificationPresenter
|
||
private let backupRequestManager: BackupRequestManager
|
||
private let backupSettingsStore: BackupSettingsStore
|
||
private let dateProvider: DateProvider
|
||
private let db: any DB
|
||
private let kvStore: KeyValueStore
|
||
private let logger: PrefixedLogger
|
||
private let notificationPresenter: NotificationPresenter
|
||
private let orphanedBackupAttachmentStore: OrphanedBackupAttachmentStore
|
||
private let remoteConfigManager: RemoteConfigManager
|
||
private let serialTaskQueue: SerialTaskQueue
|
||
private let tsAccountManager: TSAccountManager
|
||
|
||
init(
|
||
accountKeyStore: AccountKeyStore,
|
||
attachmentStore: AttachmentStore,
|
||
attachmentUploadStore: AttachmentUploadStore,
|
||
backupAttachmentDownloadProgress: BackupAttachmentDownloadProgress,
|
||
backupAttachmentDownloadStore: BackupAttachmentDownloadStore,
|
||
backupAttachmentUploadProgress: BackupAttachmentUploadProgress,
|
||
backupAttachmentUploadScheduler: BackupAttachmentUploadScheduler,
|
||
backupAttachmentUploadStore: BackupAttachmentUploadStore,
|
||
backupAttachmentUploadEraStore: BackupAttachmentUploadEraStore,
|
||
backupListMediaStore: BackupListMediaStore,
|
||
backupMediaErrorNotificationPresenter: BackupMediaErrorNotificationPresenter,
|
||
backupRequestManager: BackupRequestManager,
|
||
backupSettingsStore: BackupSettingsStore,
|
||
dateProvider: @escaping DateProvider,
|
||
db: any DB,
|
||
notificationPresenter: NotificationPresenter,
|
||
orphanedBackupAttachmentStore: OrphanedBackupAttachmentStore,
|
||
remoteConfigManager: RemoteConfigManager,
|
||
tsAccountManager: TSAccountManager,
|
||
) {
|
||
self.accountKeyStore = accountKeyStore
|
||
self.attachmentStore = attachmentStore
|
||
self.attachmentUploadStore = attachmentUploadStore
|
||
self.backupAttachmentDownloadProgress = backupAttachmentDownloadProgress
|
||
self.backupAttachmentDownloadStore = backupAttachmentDownloadStore
|
||
self.backupAttachmentUploadProgress = backupAttachmentUploadProgress
|
||
self.backupAttachmentUploadScheduler = backupAttachmentUploadScheduler
|
||
self.backupAttachmentUploadStore = backupAttachmentUploadStore
|
||
self.backupAttachmentUploadEraStore = backupAttachmentUploadEraStore
|
||
self.backupListMediaStore = backupListMediaStore
|
||
self.backupMediaErrorNotificationPresenter = backupMediaErrorNotificationPresenter
|
||
self.backupRequestManager = backupRequestManager
|
||
self.backupSettingsStore = backupSettingsStore
|
||
self.dateProvider = dateProvider
|
||
self.db = db
|
||
self.kvStore = KeyValueStore(collection: "ListBackupMediaManager")
|
||
self.logger = PrefixedLogger(prefix: "[Backups]")
|
||
self.notificationPresenter = notificationPresenter
|
||
self.orphanedBackupAttachmentStore = orphanedBackupAttachmentStore
|
||
self.remoteConfigManager = remoteConfigManager
|
||
self.serialTaskQueue = SerialTaskQueue()
|
||
self.tsAccountManager = tsAccountManager
|
||
|
||
NotificationCenter.default.addObserver(
|
||
self,
|
||
selector: #selector(backupPlanDidChange),
|
||
name: .backupPlanChanged,
|
||
object: nil,
|
||
)
|
||
}
|
||
|
||
// MARK: -
|
||
|
||
func getNeedsQueryListMedia(tx: DBReadTransaction) -> Bool {
|
||
return needsToQueryListMedia(tx: tx)
|
||
}
|
||
|
||
func wipe(tx: DBWriteTransaction) {
|
||
kvStore.removeAll(transaction: tx)
|
||
backupListMediaStore.removeAll(tx: tx)
|
||
failIfThrows {
|
||
try ListedBackupMediaObject.deleteAll(tx.database)
|
||
}
|
||
}
|
||
|
||
// MARK: -
|
||
|
||
func queryListMediaIfNeeded() async throws {
|
||
let task = serialTaskQueue.enqueue { [self] in
|
||
try await _queryListMediaIfNeeded()
|
||
}
|
||
let backgroundTask = OWSBackgroundTask(label: "[ListMediaManager]") { [task] status in
|
||
switch status {
|
||
case .expired:
|
||
task.cancel()
|
||
case .couldNotStart, .success:
|
||
break
|
||
}
|
||
}
|
||
defer { backgroundTask.end() }
|
||
try await withTaskCancellationHandler(
|
||
operation: { _ = try await task.value },
|
||
onCancel: { task.cancel() },
|
||
)
|
||
}
|
||
|
||
private func _queryListMediaIfNeeded() async throws -> ListMediaIntegrityCheckResult? {
|
||
let localAci: Aci?
|
||
let backupKey: MediaRootBackupKey?
|
||
let currentUploadEra: String
|
||
let inProgressUploadEra: String?
|
||
let inProgressStartTimestamp: UInt64?
|
||
let uploadEraOfLastListMedia: String?
|
||
let needsToQuery: Bool
|
||
let hasEverRunListMedia: Bool
|
||
let inProgressIntegrityCheckResult: ListMediaIntegrityCheckResult?
|
||
(
|
||
localAci,
|
||
backupKey,
|
||
currentUploadEra,
|
||
inProgressUploadEra,
|
||
inProgressStartTimestamp,
|
||
uploadEraOfLastListMedia,
|
||
needsToQuery,
|
||
hasEverRunListMedia,
|
||
inProgressIntegrityCheckResult,
|
||
) = db.read { tx in
|
||
return (
|
||
self.tsAccountManager.localIdentifiers(tx: tx)?.aci,
|
||
accountKeyStore.getMediaRootBackupKey(tx: tx),
|
||
backupAttachmentUploadEraStore.currentUploadEra(tx: tx),
|
||
kvStore.getString(Constants.inProgressUploadEraKey, transaction: tx),
|
||
kvStore.getUInt64(Constants.inProgressListMediaStartTimestampKey, transaction: tx),
|
||
kvStore.getString(Constants.lastListMediaUploadEraKey, transaction: tx),
|
||
needsToQueryListMedia(tx: tx),
|
||
kvStore.getBool(Constants.hasEverRunListMediaKey, defaultValue: false, transaction: tx),
|
||
kvStore.getData(Constants.inProgressIntegrityCheckResultKey, transaction: tx)
|
||
.flatMap { try? JSONDecoder().decode(ListMediaIntegrityCheckResult.self, from: $0) },
|
||
)
|
||
}
|
||
|
||
guard needsToQuery else {
|
||
return nil
|
||
}
|
||
|
||
guard let localAci else {
|
||
throw OWSAssertionError("Missing localAci!")
|
||
}
|
||
guard let backupKey else {
|
||
throw OWSAssertionError("Media backup key missing")
|
||
}
|
||
|
||
let uploadEraAtStartOfListMedia: String
|
||
let startTimestamp: UInt64
|
||
if let inProgressUploadEra, let inProgressStartTimestamp {
|
||
uploadEraAtStartOfListMedia = inProgressUploadEra
|
||
startTimestamp = inProgressStartTimestamp
|
||
} else {
|
||
startTimestamp = await db.awaitableWrite { tx in
|
||
willBeginQueryListMedia(
|
||
currentUploadEra: self.backupAttachmentUploadEraStore.currentUploadEra(tx: tx),
|
||
tx: tx,
|
||
)
|
||
}
|
||
uploadEraAtStartOfListMedia = currentUploadEra
|
||
}
|
||
|
||
func isRetryable(_ error: Error) -> Bool {
|
||
error.isNetworkFailureOrTimeout || error.is5xxServiceResponse
|
||
}
|
||
|
||
do {
|
||
// List-media is a dependency of lots of Backups-related operations,
|
||
// which means we might have many callers calling us repeatedly. To
|
||
// that end, internally retry network errors so we back off a
|
||
// healthy amount for each of those callers.
|
||
return try await Retry.performWithBackoff(
|
||
maxAttempts: 5,
|
||
isRetryable: isRetryable,
|
||
) {
|
||
try await _queryListMediaIfNeeded(
|
||
localAci: localAci,
|
||
backupKey: backupKey,
|
||
startTimestamp: startTimestamp,
|
||
uploadEraAtStartOfListMedia: uploadEraAtStartOfListMedia,
|
||
currentUploadEra: currentUploadEra,
|
||
uploadEraOfLastListMedia: uploadEraOfLastListMedia,
|
||
hasEverRunListMedia: hasEverRunListMedia,
|
||
inProgressIntegrityCheckResult: inProgressIntegrityCheckResult,
|
||
)
|
||
}
|
||
} catch is CancellationError {
|
||
throw CancellationError()
|
||
} catch where isRetryable(error) {
|
||
throw error
|
||
} catch {
|
||
logger.error("Unretryable failure in list media! \(error)")
|
||
|
||
// Post a notification so we hear about this quickly.
|
||
await backupMediaErrorNotificationPresenter.notifyIfNecessary()
|
||
|
||
// We failed for a non-retryable reason: "complete" this attempt
|
||
// so we don't make a doomed attempt for each of our callers.
|
||
await db.awaitableWrite { tx in
|
||
didFinishListMedia(
|
||
startTimestamp: startTimestamp,
|
||
integrityCheckResult: nil,
|
||
tx: tx,
|
||
)
|
||
}
|
||
|
||
throw error
|
||
}
|
||
}
|
||
|
||
private func _queryListMediaIfNeeded(
|
||
localAci: Aci,
|
||
backupKey: MediaRootBackupKey,
|
||
startTimestamp: UInt64,
|
||
uploadEraAtStartOfListMedia: String,
|
||
currentUploadEra: String,
|
||
uploadEraOfLastListMedia: String?,
|
||
hasEverRunListMedia: Bool,
|
||
inProgressIntegrityCheckResult: ListMediaIntegrityCheckResult?,
|
||
) async throws -> ListMediaIntegrityCheckResult? {
|
||
|
||
let hasCompletedListingMedia: Bool = db.read { tx in
|
||
return kvStore.getBool(
|
||
Constants.hasCompletedListingMediaKey,
|
||
defaultValue: false,
|
||
transaction: tx,
|
||
)
|
||
}
|
||
|
||
if !hasCompletedListingMedia {
|
||
try await makeListMediaRequest(backupKey: backupKey, localAci: localAci, logger: logger)
|
||
}
|
||
|
||
let hasCompletedEnumeratingAttchments: Bool = db.read { tx in
|
||
return kvStore.getBool(
|
||
Constants.hasCompletedEnumeratingAttachmentsKey,
|
||
defaultValue: false,
|
||
transaction: tx,
|
||
)
|
||
}
|
||
|
||
let integrityChecker: ListMediaIntegrityChecker
|
||
if
|
||
BuildFlags.Backups.mediaErrorDisplay,
|
||
// Skip integrity checks if we're in a new upload era, since we
|
||
// expect media to not yet be uploaded.
|
||
currentUploadEra == uploadEraOfLastListMedia
|
||
{
|
||
integrityChecker = ListMediaIntegrityCheckerImpl(
|
||
inProgressResult: inProgressIntegrityCheckResult,
|
||
listMediaStartTimestamp: startTimestamp,
|
||
uploadEraAtStartOfListMedia: uploadEraAtStartOfListMedia,
|
||
uploadEraOfLastListMedia: uploadEraOfLastListMedia,
|
||
attachmentStore: attachmentStore,
|
||
backupAttachmentUploadScheduler: backupAttachmentUploadScheduler,
|
||
backupAttachmentUploadStore: backupAttachmentUploadStore,
|
||
notificationPresenter: notificationPresenter,
|
||
orphanedBackupAttachmentStore: orphanedBackupAttachmentStore,
|
||
)
|
||
} else {
|
||
integrityChecker = ListMediaIntegrityCheckerStub()
|
||
}
|
||
|
||
if !hasCompletedEnumeratingAttchments {
|
||
let remoteConfig = remoteConfigManager.currentConfig()
|
||
|
||
struct TxContext {
|
||
let backupPlan: BackupPlan
|
||
let originalLastEnumeratedAttachmentId: Attachment.IDType?
|
||
var lastEnumeratedAttachmentId: Attachment.IDType?
|
||
var didFinish: Bool
|
||
}
|
||
|
||
try await TimeGatedBatch.processAll(
|
||
db: db,
|
||
delayTwixtTx: 0.2,
|
||
buildTxContext: { tx -> TxContext in
|
||
let lastEnumeratedAttachmentId: Attachment.IDType? = kvStore.getInt64(
|
||
Constants.lastEnumeratedAttachmentIdKey,
|
||
transaction: tx,
|
||
)
|
||
|
||
return TxContext(
|
||
backupPlan: backupSettingsStore.backupPlan(tx: tx),
|
||
originalLastEnumeratedAttachmentId: lastEnumeratedAttachmentId,
|
||
lastEnumeratedAttachmentId: lastEnumeratedAttachmentId,
|
||
didFinish: false,
|
||
)
|
||
},
|
||
processBatch: { tx, txContext throws(CancellationError) -> TimeGatedBatch.ProcessBatchResult<Void> in
|
||
if Task.isCancelled {
|
||
throw CancellationError()
|
||
}
|
||
|
||
var query = Attachment.Record
|
||
.order(Column(Attachment.Record.CodingKeys.sqliteId).asc)
|
||
.filter(Column(Attachment.Record.CodingKeys.plaintextHash) != nil)
|
||
|
||
if let id = txContext.lastEnumeratedAttachmentId {
|
||
query = query
|
||
.filter(Column(Attachment.Record.CodingKeys.sqliteId) > id)
|
||
}
|
||
|
||
guard
|
||
let attachment = failIfThrows(block: {
|
||
try query.fetchOne(tx.database)
|
||
.map { Attachment(record: $0) }
|
||
})
|
||
else {
|
||
txContext.didFinish = true
|
||
return .done(())
|
||
}
|
||
|
||
txContext.lastEnumeratedAttachmentId = attachment.id
|
||
|
||
guard let fullsizeMediaName = attachment.mediaName else {
|
||
owsFailDebug("We filtered by mediaName presence, how is it missing")
|
||
return .more
|
||
}
|
||
|
||
// Check for matches for both the fullsize and the
|
||
// thumbnail mediaId. Fullsize first.
|
||
updateAttachmentIfNeeded(
|
||
attachment: attachment,
|
||
fullsizeMediaName: fullsizeMediaName,
|
||
isThumbnail: false,
|
||
backupKey: backupKey,
|
||
uploadEraAtStartOfListMedia: uploadEraAtStartOfListMedia,
|
||
currentBackupPlan: txContext.backupPlan,
|
||
remoteConfig: remoteConfig,
|
||
hasEverRunListMedia: hasEverRunListMedia,
|
||
integrityChecker: integrityChecker,
|
||
tx: tx,
|
||
)
|
||
|
||
// Refetch the attachment to reload any mutations applied
|
||
// by the fullsize matching.
|
||
guard let attachment = attachmentStore.fetch(id: attachment.id, tx: tx) else {
|
||
owsFailDebug("Missing attachment we just fetched?")
|
||
return .more
|
||
}
|
||
|
||
updateAttachmentIfNeeded(
|
||
attachment: attachment,
|
||
fullsizeMediaName: fullsizeMediaName,
|
||
isThumbnail: true,
|
||
backupKey: backupKey,
|
||
uploadEraAtStartOfListMedia: uploadEraAtStartOfListMedia,
|
||
currentBackupPlan: txContext.backupPlan,
|
||
remoteConfig: remoteConfig,
|
||
hasEverRunListMedia: hasEverRunListMedia,
|
||
integrityChecker: integrityChecker,
|
||
tx: tx,
|
||
)
|
||
|
||
return .more
|
||
},
|
||
concludeTx: { tx, txContext in
|
||
let startAttachmentLogString = txContext.originalLastEnumeratedAttachmentId.map { String($0) } ?? "nil"
|
||
let endAttachmentLogString = txContext.lastEnumeratedAttachmentId.map { String($0) } ?? "nil"
|
||
logger.info("Checked attachments [\(startAttachmentLogString)...\(endAttachmentLogString)]. didFinish \(txContext.didFinish)")
|
||
|
||
if txContext.didFinish {
|
||
// We're done
|
||
kvStore.removeValue(forKey: Constants.lastEnumeratedAttachmentIdKey, transaction: tx)
|
||
kvStore.setBool(true, key: Constants.hasCompletedEnumeratingAttachmentsKey, transaction: tx)
|
||
} else if let lastEnumeratedAttachmentId = txContext.lastEnumeratedAttachmentId {
|
||
kvStore.setInt64(lastEnumeratedAttachmentId, key: Constants.lastEnumeratedAttachmentIdKey, transaction: tx)
|
||
}
|
||
|
||
if
|
||
let integrityCheckResult = integrityChecker.result,
|
||
let serializedResult = try? JSONEncoder().encode(integrityCheckResult)
|
||
{
|
||
kvStore.setData(
|
||
serializedResult,
|
||
key: Constants.inProgressIntegrityCheckResultKey,
|
||
transaction: tx,
|
||
)
|
||
}
|
||
},
|
||
)
|
||
}
|
||
|
||
// Any remaining attachments in the table weren't matched against a local attachment
|
||
// and should be marked for deletion.
|
||
// If we created a new attachment stream between when we checked every attachment
|
||
// above and now, that attachment will be queued for media tier upload, and that
|
||
// media tier upload job will cancel the orphan job we schedule here.
|
||
try await TimeGatedBatch.processAll(
|
||
db: db,
|
||
delayTwixtTx: 0.2,
|
||
buildTxContext: { tx -> Void in
|
||
// Nothing – we use the in-memory integrityChecker.
|
||
},
|
||
processBatch: { tx, _ throws(CancellationError) in
|
||
if Task.isCancelled {
|
||
throw CancellationError()
|
||
}
|
||
|
||
let listedMediaObject: ListedBackupMediaObject? = failIfThrows {
|
||
try ListedBackupMediaObject.fetchOne(tx.database)
|
||
}
|
||
|
||
guard let listedMediaObject else {
|
||
return .done(())
|
||
}
|
||
|
||
enqueueListedMediaForDeletion(listedMediaObject, tx: tx)
|
||
|
||
failIfThrows {
|
||
try listedMediaObject.delete(tx.database)
|
||
}
|
||
|
||
integrityChecker.updateWithOrphanedObject(
|
||
mediaId: listedMediaObject.mediaId,
|
||
backupKey: backupKey,
|
||
tx: tx,
|
||
)
|
||
|
||
return .more
|
||
},
|
||
concludeTx: { tx, _ in
|
||
if
|
||
let integrityCheckResult = integrityChecker.result,
|
||
let serializedResult = try? JSONEncoder().encode(integrityCheckResult)
|
||
{
|
||
kvStore.setData(serializedResult, key: Constants.inProgressIntegrityCheckResultKey, transaction: tx)
|
||
}
|
||
},
|
||
)
|
||
|
||
let needsToRunAgain = await db.awaitableWrite { tx in
|
||
self.didFinishListMedia(startTimestamp: startTimestamp, integrityCheckResult: integrityChecker.result, tx: tx)
|
||
return needsToQueryListMedia(tx: tx)
|
||
}
|
||
|
||
integrityChecker.logAndNotifyIfNeeded()
|
||
|
||
if needsToRunAgain {
|
||
// Return the first integrity check result, not the second, because
|
||
// usually earlier results are more interesting. Once we run list
|
||
// media once, we've already synced local and remote state.
|
||
_ = try await _queryListMediaIfNeeded()
|
||
}
|
||
return integrityChecker.result
|
||
}
|
||
|
||
// MARK: Remote attachment mapping
|
||
|
||
private struct ListedBackupMediaObject: Codable, FetchableRecord, MutablePersistableRecord {
|
||
// SQLite row id
|
||
var id: Int64?
|
||
// Either fullsize or thumbnail media id; the server doesn't know.
|
||
let mediaId: Data
|
||
let cdnNumber: UInt32
|
||
// Size on the cdn according to the server
|
||
let objectLength: UInt32
|
||
|
||
init(
|
||
mediaId: Data,
|
||
cdnNumber: UInt32,
|
||
objectLength: UInt32,
|
||
) {
|
||
self.mediaId = mediaId
|
||
self.cdnNumber = cdnNumber
|
||
self.objectLength = objectLength
|
||
}
|
||
|
||
static var databaseTableName: String { "ListedBackupMediaObject" }
|
||
|
||
mutating func didInsert(with rowID: Int64, for column: String?) {
|
||
self.id = rowID
|
||
}
|
||
|
||
enum CodingKeys: CodingKey {
|
||
case id
|
||
case mediaId
|
||
case cdnNumber
|
||
case objectLength
|
||
}
|
||
}
|
||
|
||
/// Query the list media endpoint, building the ListedBackupMediaObject table in the databse.
|
||
///
|
||
/// The server lists media by mediaId, which we do not store because it is derived from the
|
||
/// mediaName via the backup key (which can change). Therefore, to match against local
|
||
/// attachments we need to index over them all and derive their mediaIds, and match against
|
||
/// the already-persisted server objects.
|
||
private func makeListMediaRequest(
|
||
backupKey: MediaRootBackupKey,
|
||
localAci: Aci,
|
||
logger: PrefixedLogger,
|
||
) async throws {
|
||
let backupAuth: BackupServiceAuth = try await backupRequestManager.fetchBackupServiceAuth(
|
||
for: backupKey,
|
||
localAci: localAci,
|
||
auth: .implicit(),
|
||
logger: logger,
|
||
)
|
||
|
||
var nextCursor: String? = db.read { tx in
|
||
return kvStore.getString(Constants.paginationCursorKey, transaction: tx)
|
||
}
|
||
|
||
while true {
|
||
try Task.checkCancellation()
|
||
|
||
let page = try await backupRequestManager.listMediaObjects(
|
||
cursor: nextCursor,
|
||
limit: nil, /* let the server determine the page size */
|
||
auth: backupAuth,
|
||
logger: logger,
|
||
)
|
||
|
||
await persistListedMediaPage(page)
|
||
|
||
if let cursor = page.cursor {
|
||
nextCursor = cursor
|
||
} else {
|
||
// Done
|
||
return
|
||
}
|
||
}
|
||
}
|
||
|
||
private func persistListedMediaPage(
|
||
_ page: BackupArchive.Response.ListMediaResult,
|
||
) async {
|
||
await db.awaitableWrite { tx in
|
||
for listedMediaObject in page.storedMediaObjects {
|
||
guard let mediaId = try? Data.data(fromBase64Url: listedMediaObject.mediaId) else {
|
||
owsFailDebug("Invalid mediaId from server!")
|
||
continue
|
||
}
|
||
guard let objectLength = UInt32(exactly: listedMediaObject.objectLength) else {
|
||
owsFailDebug("Listed object too large!")
|
||
continue
|
||
}
|
||
var record = ListedBackupMediaObject(
|
||
mediaId: mediaId,
|
||
cdnNumber: listedMediaObject.cdn,
|
||
objectLength: objectLength,
|
||
)
|
||
|
||
failIfThrows {
|
||
try record.insert(tx.database)
|
||
}
|
||
}
|
||
if let cursor = page.cursor {
|
||
kvStore.setString(
|
||
cursor,
|
||
key: Constants.paginationCursorKey,
|
||
transaction: tx,
|
||
)
|
||
} else {
|
||
// We've reached the last page, mark complete.
|
||
kvStore.removeValue(forKey: Constants.paginationCursorKey, transaction: tx)
|
||
kvStore.setBool(
|
||
true,
|
||
key: Constants.hasCompletedListingMediaKey,
|
||
transaction: tx,
|
||
)
|
||
}
|
||
}
|
||
}
|
||
|
||
// MARK: Per-Attachment handling
|
||
|
||
/// Given an attachment, match it against any listed media in the
|
||
/// ListedBackupMediaObject table, and update it as needed.
|
||
///
|
||
/// - parameter uploadEraAtStartOfListMedia: The most we can guarantee is
|
||
/// that the listed cdn info is accurate as of the upload era we had when we started listing
|
||
/// media. Because the request is paginated (and this whole job is durable), the upload era
|
||
/// may have since changed, which may make the cdn info now invalid. We will still use
|
||
/// maybe-outdated cdn info at download time; we just don't want to overpromise and assume
|
||
/// its using the uploadEra as of the time we process he listed media.
|
||
/// - parameter currentBackupPlan: Unlike upload era, we want this backup plan to
|
||
/// be the latest as of processing time; we will enqueue.dequeue uploads/downloads based
|
||
/// on backupPlan so whatever the backupPlan was when we started list media doesn't matter,
|
||
/// we upload and download _now_ based on plan state _now_.
|
||
private func updateAttachmentIfNeeded(
|
||
attachment: Attachment,
|
||
fullsizeMediaName: String,
|
||
isThumbnail: Bool,
|
||
backupKey: MediaRootBackupKey,
|
||
uploadEraAtStartOfListMedia: String,
|
||
currentBackupPlan: BackupPlan,
|
||
remoteConfig: RemoteConfig,
|
||
hasEverRunListMedia: Bool,
|
||
integrityChecker: ListMediaIntegrityChecker,
|
||
tx: DBWriteTransaction,
|
||
) {
|
||
// Either the fullsize or the thumbnail media name
|
||
let mediaName: String
|
||
// Either the fullsize of the thumbnail cdn number if we have it
|
||
let localCdnNumber: UInt32?
|
||
if isThumbnail {
|
||
mediaName = AttachmentBackupThumbnail.thumbnailMediaName(fullsizeMediaName: fullsizeMediaName)
|
||
localCdnNumber = attachment.thumbnailMediaTierInfo?.cdnNumber
|
||
} else {
|
||
mediaName = fullsizeMediaName
|
||
localCdnNumber = attachment.mediaTierInfo?.cdnNumber
|
||
}
|
||
|
||
let mediaId: Data
|
||
do {
|
||
mediaId = try backupKey.deriveMediaId(mediaName)
|
||
} catch {
|
||
owsFailDebug("Failed to derive mediaID for mediaName!")
|
||
return
|
||
}
|
||
|
||
let matchedListedMediasQuery = ListedBackupMediaObject
|
||
.filter(Column(ListedBackupMediaObject.CodingKeys.mediaId) == mediaId)
|
||
.order(Column(ListedBackupMediaObject.CodingKeys.id).asc)
|
||
let matchedListedMedias = failIfThrows {
|
||
try matchedListedMediasQuery.fetchAll(tx.database)
|
||
}
|
||
|
||
guard
|
||
let matchedListedMedia = preferredListedMedia(
|
||
matchedListedMedias,
|
||
localCdnNumber: localCdnNumber,
|
||
remoteConfig: remoteConfig,
|
||
)
|
||
else {
|
||
// Call this _before_ we update the attachment; we want to check
|
||
// against the old local state vs remote state.
|
||
integrityChecker.updateWithUnuploadedAttachment(
|
||
attachment: attachment,
|
||
isFullsize: !isThumbnail,
|
||
tx: tx,
|
||
)
|
||
|
||
// No listed media matched our local attachment.
|
||
// Mark media tier info (if any) as expired.
|
||
markMediaTierUploadExpiredIfNeeded(
|
||
attachment,
|
||
isThumbnail: isThumbnail,
|
||
currentBackupPlan: currentBackupPlan,
|
||
uploadEraAtStartOfListMedia: uploadEraAtStartOfListMedia,
|
||
remoteConfig: remoteConfig,
|
||
hasEverRunListMedia: hasEverRunListMedia,
|
||
tx: tx,
|
||
)
|
||
return
|
||
}
|
||
|
||
// Call this _before_ we update the attachment; we want to check
|
||
// against the old local state vs remote state.
|
||
integrityChecker.updateWithUploadedAttachment(
|
||
attachment: attachment,
|
||
isFullsize: !isThumbnail,
|
||
remoteCdnNumber: matchedListedMedia.cdnNumber,
|
||
tx: tx,
|
||
)
|
||
|
||
updateWithListedCdn(
|
||
attachment,
|
||
listedMedia: matchedListedMedia,
|
||
isThumbnail: isThumbnail,
|
||
uploadEraAtStartOfListMedia: uploadEraAtStartOfListMedia,
|
||
currentBackupPlan: currentBackupPlan,
|
||
remoteConfig: remoteConfig,
|
||
tx: tx,
|
||
)
|
||
|
||
// Clear out the matched listed media row so we don't
|
||
// mark the upload for deletion later.
|
||
failIfThrows {
|
||
try matchedListedMedia.delete(tx.database)
|
||
}
|
||
}
|
||
|
||
/// It is possible (though unusual) to end up with the same object (same mediaId)
|
||
/// on multiple CDNs. (e.g. if we delete an attachnent then having it forwarded to
|
||
/// us again with the same encryption key, then we reupload to media tier and get
|
||
/// a different CDN number).
|
||
///
|
||
/// This method, given an array of listed media objects with the same mediaId,
|
||
/// returns the preferred object to keep, with the rest being eligible to be deleted
|
||
/// from the media tier.
|
||
///
|
||
/// In general we want to keep the most "recent" CDN number; the one the server
|
||
/// most recently gave us in an upload form and therefore the most up to date.
|
||
/// We don't know this directly, but if our local copy of an attachment has a cdn
|
||
/// number on it, that means its the most recent upload _this device_ knows about,
|
||
/// so we prefer that. Otherwise let the server choose by picking the one in our
|
||
/// remote config or, lastly, the first one the list media endpoint gave us.
|
||
private func preferredListedMedia(
|
||
_ listedMedias: [ListedBackupMediaObject],
|
||
localCdnNumber: UInt32?,
|
||
remoteConfig: RemoteConfig,
|
||
) -> ListedBackupMediaObject? {
|
||
var preferredListedMedia: ListedBackupMediaObject?
|
||
for listedMedia in listedMedias {
|
||
if listedMedia.cdnNumber == localCdnNumber {
|
||
// Always prefer the one matching the local cdn number,
|
||
// if we have one, on the assumption that the local value
|
||
// represents the most recent upload (the upload on this
|
||
// current, registered device), and therefore the most
|
||
// recent determination by the server of which CDN to use.
|
||
return listedMedia
|
||
}
|
||
if listedMedia.cdnNumber == remoteConfig.mediaTierFallbackCdnNumber {
|
||
// Prefer the remote config cdn number, as we can at least
|
||
// somewhat control this remotely.
|
||
preferredListedMedia = listedMedia
|
||
} else if preferredListedMedia == nil {
|
||
// Otherwise take the first one given to us by the server.
|
||
preferredListedMedia = listedMedia
|
||
}
|
||
}
|
||
return preferredListedMedia
|
||
}
|
||
|
||
/// We have a local attachment not represented in listed media;
|
||
/// mark any media tier info as expired/invalid/gone.
|
||
private func markMediaTierUploadExpiredIfNeeded(
|
||
_ attachment: Attachment,
|
||
isThumbnail: Bool,
|
||
currentBackupPlan: BackupPlan,
|
||
uploadEraAtStartOfListMedia: String,
|
||
remoteConfig: RemoteConfig,
|
||
hasEverRunListMedia: Bool,
|
||
tx: DBWriteTransaction,
|
||
) {
|
||
if isThumbnail, let thumbnailMediaTierInfo = attachment.thumbnailMediaTierInfo {
|
||
if thumbnailMediaTierInfo.uploadEra == uploadEraAtStartOfListMedia {
|
||
logger.warn("Unexpectedly missing thumbnail we thought was on media tier cdn \(attachment.id)")
|
||
} else {
|
||
// The uploadEra has rotated, so it's reasonable that the
|
||
// attachment is un-uploaded.
|
||
}
|
||
|
||
attachmentStore.removeThumbnailMediaTierInfo(
|
||
attachment: attachment,
|
||
tx: tx,
|
||
)
|
||
}
|
||
|
||
if !isThumbnail, let mediaTierInfo = attachment.mediaTierInfo {
|
||
if mediaTierInfo.uploadEra == uploadEraAtStartOfListMedia {
|
||
logger.warn("Unexpectedly missing fullsize we thought was on media tier cdn \(attachment.id)")
|
||
} else {
|
||
// The uploadEra has rotated, so it's reasonable that the
|
||
// attachment is un-uploaded.
|
||
}
|
||
|
||
attachmentStore.removeMediaTierInfo(
|
||
attachment: attachment,
|
||
tx: tx,
|
||
)
|
||
}
|
||
|
||
// If the media tier upload we had was expired, we need to
|
||
// reupload, so enqueue that.
|
||
// Note: we enqueue uploads on non-primary devices; the uploads
|
||
// just won't be run.
|
||
backupAttachmentUploadScheduler.enqueueUsingHighestPriorityOwnerIfNeeded(
|
||
attachment,
|
||
mode: isThumbnail ? .thumbnail : .fullsize,
|
||
tx: tx,
|
||
)
|
||
|
||
if
|
||
let existingDownload = backupAttachmentDownloadStore.getEnqueuedDownload(
|
||
attachmentRowId: attachment.id,
|
||
thumbnail: isThumbnail,
|
||
tx: tx,
|
||
)
|
||
{
|
||
cancelEnqueuedDownload(
|
||
existingDownload,
|
||
for: attachment,
|
||
isThumbnail: isThumbnail,
|
||
currentBackupPlan: currentBackupPlan,
|
||
remoteConfig: remoteConfig,
|
||
tx: tx,
|
||
)
|
||
}
|
||
}
|
||
|
||
private func cancelEnqueuedDownload(
|
||
_ existingDownload: QueuedBackupAttachmentDownload,
|
||
for attachment: Attachment,
|
||
isThumbnail: Bool,
|
||
currentBackupPlan: BackupPlan,
|
||
remoteConfig: RemoteConfig,
|
||
tx: DBWriteTransaction,
|
||
) {
|
||
backupAttachmentDownloadStore.remove(
|
||
attachmentId: attachment.id,
|
||
thumbnail: isThumbnail,
|
||
tx: tx,
|
||
)
|
||
// Mark the cancelled download as "finished" because it was cancelled,
|
||
// but we want the progress bar to complete.
|
||
var shouldMarkDownloadProgressFinished = true
|
||
if
|
||
!isThumbnail,
|
||
let transitTierEligibilityState = BackupAttachmentDownloadEligibility.transitTierFullsizeState(
|
||
attachment: attachment,
|
||
mostRecentReferenceTimestamp: existingDownload.maxOwnerTimestamp,
|
||
currentTimestamp: dateProvider().ows_millisecondsSince1970,
|
||
remoteConfig: remoteConfig,
|
||
backupPlan: currentBackupPlan,
|
||
isPrimaryDevice: true, // Only primaries run list-media
|
||
)
|
||
{
|
||
// We just found we can't download from media tier, but
|
||
// fullsize downloads can also come from transit tier (and
|
||
// are represented by the same download row). If indeed eligible,
|
||
// re-enqueue as just a transit tier download.
|
||
var existingDownload = existingDownload
|
||
existingDownload.id = nil
|
||
existingDownload.canDownloadFromMediaTier = false
|
||
existingDownload.state = transitTierEligibilityState
|
||
failIfThrows {
|
||
try existingDownload.insert(tx.database)
|
||
}
|
||
shouldMarkDownloadProgressFinished = false
|
||
}
|
||
if shouldMarkDownloadProgressFinished {
|
||
tx.addSyncCompletion {
|
||
if !isThumbnail {
|
||
Task {
|
||
await self.backupAttachmentDownloadProgress.didFinishDownloadOfFullsizeAttachment(
|
||
withId: attachment.id,
|
||
byteCount: UInt64(QueuedBackupAttachmentDownload.estimatedByteCount(
|
||
attachment: attachment,
|
||
reference: nil,
|
||
isThumbnail: isThumbnail,
|
||
canDownloadFromMediaTier: true,
|
||
)),
|
||
)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Update a local attachment with matched listed media cdn info.
|
||
/// The local attachment may or may not already have media tier
|
||
/// cdn information; it will be overwritten.
|
||
private func updateWithListedCdn(
|
||
_ attachment: Attachment,
|
||
listedMedia: ListedBackupMediaObject,
|
||
isThumbnail: Bool,
|
||
uploadEraAtStartOfListMedia: String,
|
||
currentBackupPlan: BackupPlan,
|
||
remoteConfig: RemoteConfig,
|
||
tx: DBWriteTransaction,
|
||
) {
|
||
// Update the attachment itself.
|
||
let didSetCdnInfo = updateCdnInfoIfPossible(
|
||
of: attachment,
|
||
from: listedMedia,
|
||
isThumbnail: isThumbnail,
|
||
uploadEraAtStartOfListMedia: uploadEraAtStartOfListMedia,
|
||
tx: tx,
|
||
)
|
||
|
||
var attachment = attachment
|
||
if didSetCdnInfo {
|
||
// Refetch the attachment so we have the latest info.
|
||
guard let refetched = attachmentStore.fetch(id: attachment.id, tx: tx) else {
|
||
owsFailDebug("How is this attachment gone?")
|
||
return
|
||
}
|
||
attachment = refetched
|
||
}
|
||
|
||
// Since we now know this is uploaded, we can go ahead and remove
|
||
// from the upload queue if present.
|
||
if
|
||
let finishedRecord = backupAttachmentUploadStore.markUploadDone(
|
||
for: attachment.id,
|
||
fullsize: !isThumbnail,
|
||
tx: tx,
|
||
file: nil,
|
||
function: nil,
|
||
line: nil,
|
||
)
|
||
{
|
||
logger.info("Marked discovered attachment \(attachment.id) done. fullsize? \(!isThumbnail)")
|
||
if finishedRecord.isFullsize {
|
||
Task {
|
||
await backupAttachmentUploadProgress.didFinishUploadOfFullsizeAttachment(
|
||
uploadRecord: finishedRecord,
|
||
)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Enqueue a download from the newly-discovered cdn info.
|
||
// If it was already enqueued, won't hurt anything.
|
||
enqueueDownloadIfNeeded(
|
||
attachment: attachment,
|
||
isThumbnail: isThumbnail,
|
||
currentBackupPlan: currentBackupPlan,
|
||
remoteConfig: remoteConfig,
|
||
tx: tx,
|
||
)
|
||
}
|
||
|
||
/// - returns True if cdn info was set
|
||
private func updateCdnInfoIfPossible(
|
||
of attachment: Attachment,
|
||
from listedMedia: ListedBackupMediaObject,
|
||
isThumbnail: Bool,
|
||
uploadEraAtStartOfListMedia: String,
|
||
tx: DBWriteTransaction,
|
||
) -> Bool {
|
||
if isThumbnail {
|
||
// Thumbnails are easy; no additional metadata is needed.
|
||
attachmentStore.saveMediaTierThumbnailInfo(
|
||
attachment: attachment,
|
||
thumbnailMediaTierInfo: Attachment.ThumbnailMediaTierInfo(
|
||
cdnNumber: listedMedia.cdnNumber,
|
||
uploadEra: uploadEraAtStartOfListMedia,
|
||
lastDownloadAttemptTimestamp: nil,
|
||
),
|
||
tx: tx,
|
||
)
|
||
return true
|
||
}
|
||
// In order for the fullsize attachment to download, we need some metadata.
|
||
// We might have this either from a local stream (if we matched against
|
||
// the media name/id we generated locally) or from a restored backup (if
|
||
// we matched against the media name/id we pulled off the backup proto).
|
||
let fullsizeUnencryptedByteCount = attachment.mediaTierInfo?.unencryptedByteCount
|
||
?? attachment.streamInfo?.unencryptedByteCount
|
||
let fullsizePlaintextHash = attachment.mediaTierInfo?.plaintextHash
|
||
?? attachment.streamInfo?.plaintextHash
|
||
?? attachment.plaintextHash
|
||
guard
|
||
let fullsizeUnencryptedByteCount,
|
||
let fullsizePlaintextHash
|
||
else {
|
||
// We have a matching local attachment but we don't have
|
||
// sufficient metadata from either a backup or local stream
|
||
// to be able to download, anyway. Schedule the upload for
|
||
// deletion, its unuseable. This should never happen*, because
|
||
// how would we have a media id to match against but lack the
|
||
// other info?
|
||
// * never, unless we trigger a manual list media before
|
||
// OrphanedBackupAttachmentManager finishes.
|
||
logger.error("Missing media tier metadata but matched by media id somehow")
|
||
enqueueListedMediaForDeletion(listedMedia, tx: tx)
|
||
return false
|
||
}
|
||
|
||
attachmentStore.saveMediaTierInfo(
|
||
attachment: attachment,
|
||
mediaTierInfo: Attachment.MediaTierInfo(
|
||
cdnNumber: listedMedia.cdnNumber,
|
||
unencryptedByteCount: fullsizeUnencryptedByteCount,
|
||
plaintextHash: fullsizePlaintextHash,
|
||
incrementalMacInfo: attachment.mediaTierInfo?.incrementalMacInfo,
|
||
uploadEra: uploadEraAtStartOfListMedia,
|
||
lastDownloadAttemptTimestamp: nil,
|
||
),
|
||
tx: tx,
|
||
)
|
||
return true
|
||
}
|
||
|
||
private func enqueueDownloadIfNeeded(
|
||
attachment: Attachment,
|
||
isThumbnail: Bool,
|
||
currentBackupPlan: BackupPlan,
|
||
remoteConfig: RemoteConfig,
|
||
tx: DBWriteTransaction,
|
||
) {
|
||
guard
|
||
let mostRecentReference = attachmentStore.fetchMostRecentReference(
|
||
toAttachmentId: attachment.id,
|
||
tx: tx,
|
||
)
|
||
else {
|
||
return
|
||
}
|
||
|
||
let currentTimestamp = dateProvider().ows_millisecondsSince1970
|
||
|
||
// We check only media tier eligibility, as that's what may have changed
|
||
// as a result of list media. The attachment may already have been eligible
|
||
// for transit tier download; we will just overwrite the already enqueued download.
|
||
let mediaTierDownloadState: QueuedBackupAttachmentDownload.State?
|
||
// But to actually enqueue, we want the combined transit + media tier state
|
||
// so that we don't overwrite existing transit tier state incorrectly.
|
||
let combinedDownloadState: QueuedBackupAttachmentDownload.State?
|
||
if isThumbnail {
|
||
mediaTierDownloadState = BackupAttachmentDownloadEligibility.mediaTierThumbnailState(
|
||
attachment: attachment,
|
||
backupPlan: currentBackupPlan,
|
||
mostRecentReferenceTimestamp: {
|
||
switch mostRecentReference.owner {
|
||
case .message(let messageSource):
|
||
return messageSource.receivedAtTimestamp
|
||
case .thread, .storyMessage:
|
||
return nil
|
||
}
|
||
}(),
|
||
currentTimestamp: currentTimestamp,
|
||
)
|
||
combinedDownloadState = mediaTierDownloadState
|
||
} else {
|
||
let eligibility = BackupAttachmentDownloadEligibility.forAttachment(
|
||
attachment,
|
||
mostRecentReference: mostRecentReference,
|
||
currentTimestamp: currentTimestamp,
|
||
backupPlan: currentBackupPlan,
|
||
remoteConfig: remoteConfig,
|
||
isPrimaryDevice: true, // Only primaries run list-media
|
||
)
|
||
mediaTierDownloadState = eligibility.fullsizeMediaTierState
|
||
combinedDownloadState = eligibility.fullsizeState
|
||
}
|
||
|
||
guard let mediaTierDownloadState, let combinedDownloadState else {
|
||
// Not possible to download.
|
||
return
|
||
}
|
||
|
||
var state = combinedDownloadState
|
||
switch mediaTierDownloadState {
|
||
case .done:
|
||
// Don't bother enqueueing if already done.
|
||
return
|
||
case .ineligible:
|
||
// Its ineligible now due to backupPlan state, but we should
|
||
// still enqueue it (as ineligible) so it can become ready later
|
||
// if backupPlan state changes.
|
||
state = .ineligible
|
||
fallthrough
|
||
case .ready:
|
||
// Dequeue any existing download first; this will reset the retry counter
|
||
backupAttachmentDownloadStore.remove(
|
||
attachmentId: attachment.id,
|
||
thumbnail: isThumbnail,
|
||
tx: tx,
|
||
)
|
||
|
||
backupAttachmentDownloadStore.enqueue(
|
||
ReferencedAttachment(
|
||
reference: mostRecentReference,
|
||
attachment: attachment,
|
||
),
|
||
thumbnail: isThumbnail,
|
||
// We got here because we discovered we can download
|
||
// from media tier, that's the whole point.
|
||
canDownloadFromMediaTier: true,
|
||
state: state,
|
||
currentTimestamp: currentTimestamp,
|
||
tx: tx,
|
||
)
|
||
}
|
||
}
|
||
|
||
private func enqueueListedMediaForDeletion(
|
||
_ listedMedia: ListedBackupMediaObject,
|
||
tx: DBWriteTransaction,
|
||
) {
|
||
var orphanRecord = OrphanedBackupAttachment.discoveredOnServer(
|
||
cdnNumber: listedMedia.cdnNumber,
|
||
mediaId: listedMedia.mediaId,
|
||
)
|
||
orphanedBackupAttachmentStore.insert(&orphanRecord, tx: tx)
|
||
}
|
||
|
||
// MARK: State
|
||
|
||
private func needsToQueryListMedia(tx: DBReadTransaction) -> Bool {
|
||
guard tsAccountManager.registrationState(tx: tx).isRegisteredPrimaryDevice else {
|
||
return false
|
||
}
|
||
|
||
switch backupSettingsStore.backupPlan(tx: tx) {
|
||
case .disabled, .disabling, .free:
|
||
return false
|
||
case .paid, .paidExpiringSoon, .paidAsTester:
|
||
break
|
||
}
|
||
|
||
if kvStore.getString(Constants.inProgressUploadEraKey, transaction: tx) != nil {
|
||
return true
|
||
}
|
||
|
||
guard let lastQueriedUploadEra = kvStore.getString(Constants.lastListMediaUploadEraKey, transaction: tx) else {
|
||
// We've never run list-media on this device! Do so now. (This
|
||
// ensures we run a list-media after restoring onto a new device.)
|
||
return true
|
||
}
|
||
|
||
if backupAttachmentUploadEraStore.currentUploadEra(tx: tx) != lastQueriedUploadEra {
|
||
return true
|
||
}
|
||
|
||
if backupListMediaStore.getManualNeedsListMedia(tx: tx) {
|
||
return true
|
||
}
|
||
|
||
// As a catch-all defense against bugs or whatever else, periodically
|
||
// query to make sure our local state is in sync with the server.
|
||
let nextPeriodicListMediaDate: Date = {
|
||
guard
|
||
let lastListMediaDate = kvStore
|
||
.getUInt64(Constants.lastListMediaStartTimestampKey, transaction: tx)
|
||
.map({ Date(millisecondsSince1970: $0) })
|
||
else {
|
||
return .distantPast
|
||
}
|
||
|
||
let remoteConfig = remoteConfigManager.currentConfig()
|
||
let refreshInterval: TimeInterval
|
||
if backupSettingsStore.hasConsumedMediaTierCapacity(tx: tx) {
|
||
refreshInterval = remoteConfig.backupListMediaOutOfQuotaRefreshInterval
|
||
} else {
|
||
refreshInterval = remoteConfig.backupListMediaDefaultRefreshInterval
|
||
}
|
||
|
||
return lastListMediaDate.addingTimeInterval(refreshInterval)
|
||
}()
|
||
|
||
return dateProvider() > nextPeriodicListMediaDate
|
||
}
|
||
|
||
/// Returns start timestamp for this run
|
||
private func willBeginQueryListMedia(
|
||
currentUploadEra: String,
|
||
tx: DBWriteTransaction,
|
||
) -> UInt64 {
|
||
let startTimestamp = dateProvider().ows_millisecondsSince1970
|
||
|
||
if kvStore.getString(Constants.inProgressUploadEraKey, transaction: tx) != nil {
|
||
guard let startTimestamp = kvStore.getUInt64(Constants.inProgressListMediaStartTimestampKey, transaction: tx) else {
|
||
owsFailDebug("Missing start timestamp!")
|
||
return startTimestamp
|
||
}
|
||
return startTimestamp
|
||
}
|
||
|
||
failIfThrows {
|
||
try ListedBackupMediaObject.deleteAll(tx.database)
|
||
}
|
||
|
||
self.kvStore.setString(currentUploadEra, key: Constants.inProgressUploadEraKey, transaction: tx)
|
||
self.kvStore.setUInt64(
|
||
startTimestamp,
|
||
key: Constants.inProgressListMediaStartTimestampKey,
|
||
transaction: tx,
|
||
)
|
||
return startTimestamp
|
||
}
|
||
|
||
private func didFinishListMedia(
|
||
startTimestamp: UInt64,
|
||
integrityCheckResult: ListMediaIntegrityCheckResult?,
|
||
tx: DBWriteTransaction,
|
||
) {
|
||
self.kvStore.setBool(true, key: Constants.hasEverRunListMediaKey, transaction: tx)
|
||
if let uploadEra = kvStore.getString(Constants.inProgressUploadEraKey, transaction: tx) {
|
||
self.kvStore.setString(uploadEra, key: Constants.lastListMediaUploadEraKey, transaction: tx)
|
||
self.kvStore.removeValue(forKey: Constants.inProgressUploadEraKey, transaction: tx)
|
||
} else {
|
||
owsFailDebug("Missing in progress upload era?")
|
||
}
|
||
self.kvStore.setUInt64(startTimestamp, key: Constants.lastListMediaStartTimestampKey, transaction: tx)
|
||
backupListMediaStore.setManualNeedsListMedia(false, tx: tx)
|
||
kvStore.removeValue(forKey: Constants.inProgressListMediaStartTimestampKey, transaction: tx)
|
||
|
||
if let integrityCheckResult {
|
||
if integrityCheckResult.hasFailures {
|
||
backupListMediaStore.setLastFailingIntegrityCheckResult(integrityCheckResult, tx: tx)
|
||
}
|
||
backupListMediaStore.setMostRecentIntegrityCheckResult(integrityCheckResult, tx: tx)
|
||
}
|
||
kvStore.removeValue(forKey: Constants.inProgressIntegrityCheckResultKey, transaction: tx)
|
||
|
||
self.kvStore.setBool(false, key: Constants.hasCompletedListingMediaKey, transaction: tx)
|
||
kvStore.removeValue(forKey: Constants.paginationCursorKey, transaction: tx)
|
||
self.kvStore.setBool(false, key: Constants.hasCompletedEnumeratingAttachmentsKey, transaction: tx)
|
||
self.kvStore.removeValue(forKey: Constants.lastEnumeratedAttachmentIdKey, transaction: tx)
|
||
}
|
||
|
||
@objc
|
||
private func backupPlanDidChange() {
|
||
Task {
|
||
switch self.db.read(block: backupSettingsStore.backupPlan(tx:)) {
|
||
case .free, .paid, .paidAsTester, .paidExpiringSoon, .disabling:
|
||
return
|
||
case .disabled:
|
||
// Rotate the last integrity check failure when disabled
|
||
await self.db.awaitableWrite { tx in
|
||
backupListMediaStore.setLastFailingIntegrityCheckResult(nil, tx: tx)
|
||
backupListMediaStore.setMostRecentIntegrityCheckResult(nil, tx: tx)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
private enum Constants {
|
||
/// Maps to the upload era (active subscription) when we last queried the list media
|
||
/// endpoint, or nil if its never been queried.
|
||
static let lastListMediaUploadEraKey = "lastListMediaUploadEra"
|
||
|
||
/// Maps to the timestamp we last completed a list media request.
|
||
static let lastListMediaStartTimestampKey = "lastListMediaTimestamp"
|
||
static let inProgressListMediaStartTimestampKey = "inProgressListMediaTimestamp"
|
||
|
||
/// True if we've ever run list media in the lifetime of this app.
|
||
static let hasEverRunListMediaKey = "hasEverRunListMedia"
|
||
|
||
/// If there is a list media in progress, the value at this key is the upload era that was set
|
||
/// at the start of that in progress run.
|
||
static let inProgressUploadEraKey = "inProgressUploadEraKey"
|
||
|
||
/// If we have finished all pages of the list media request, hasCompletedPaginationKey's value
|
||
/// will be true. If not, paginationCursorKey points to the cursor provided by the server on the last
|
||
/// page, or nil if no pages have finished processing yet.
|
||
static let paginationCursorKey = "paginationCursorKey"
|
||
static let hasCompletedListingMediaKey = "hasCompletedListingMediaKey"
|
||
|
||
/// If we have finished enumerating all attachments to compare to listed media,
|
||
/// hasCompletedEnumeratingAttachmentsKey''s value will be true.
|
||
/// If not, lastEnumeratedAttachmentIdKey's value is the last attachment id enumerated,
|
||
/// or nil if no attachments have been enumerated yet.
|
||
static let lastEnumeratedAttachmentIdKey = "lastEnumeratedAttachmentIdKey"
|
||
static let hasCompletedEnumeratingAttachmentsKey = "hasCompletedEnumeratingAttachmentsKey"
|
||
|
||
static let inProgressIntegrityCheckResultKey = "inProgressIntegrityCheckResultKey"
|
||
}
|
||
}
|
||
|
||
// MARK: -
|
||
|
||
private protocol ListMediaIntegrityChecker {
|
||
|
||
func updateWithUnuploadedAttachment(
|
||
attachment: Attachment,
|
||
isFullsize: Bool,
|
||
tx: DBReadTransaction,
|
||
)
|
||
|
||
func updateWithUploadedAttachment(
|
||
attachment: Attachment,
|
||
isFullsize: Bool,
|
||
remoteCdnNumber: UInt32,
|
||
tx: DBReadTransaction,
|
||
)
|
||
|
||
func updateWithOrphanedObject(
|
||
mediaId: Data,
|
||
backupKey: MediaRootBackupKey,
|
||
tx: DBReadTransaction,
|
||
)
|
||
|
||
var result: ListMediaIntegrityCheckResult? { get }
|
||
|
||
func logAndNotifyIfNeeded()
|
||
}
|
||
|
||
private class ListMediaIntegrityCheckerImpl: ListMediaIntegrityChecker {
|
||
|
||
var _result: ListMediaIntegrityCheckResult
|
||
|
||
private let uploadEraAtStartOfListMedia: String
|
||
private let uploadEraOfLastListMedia: String?
|
||
|
||
private let attachmentStore: AttachmentStore
|
||
private let backupAttachmentUploadScheduler: BackupAttachmentUploadScheduler
|
||
private let backupAttachmentUploadStore: BackupAttachmentUploadStore
|
||
private let logger: PrefixedLogger
|
||
private let notificationPresenter: NotificationPresenter
|
||
private let orphanedBackupAttachmentStore: OrphanedBackupAttachmentStore
|
||
|
||
init(
|
||
inProgressResult: ListMediaIntegrityCheckResult?,
|
||
listMediaStartTimestamp: UInt64,
|
||
uploadEraAtStartOfListMedia: String,
|
||
uploadEraOfLastListMedia: String?,
|
||
attachmentStore: AttachmentStore,
|
||
backupAttachmentUploadScheduler: BackupAttachmentUploadScheduler,
|
||
backupAttachmentUploadStore: BackupAttachmentUploadStore,
|
||
notificationPresenter: NotificationPresenter,
|
||
orphanedBackupAttachmentStore: OrphanedBackupAttachmentStore,
|
||
) {
|
||
self.uploadEraAtStartOfListMedia = uploadEraAtStartOfListMedia
|
||
self.uploadEraOfLastListMedia = uploadEraOfLastListMedia
|
||
self._result = inProgressResult ?? ListMediaIntegrityCheckResult(
|
||
listMediaStartTimestamp: listMediaStartTimestamp,
|
||
fullsize: .empty,
|
||
thumbnail: .empty,
|
||
orphanedObjectCount: 0,
|
||
)
|
||
self.attachmentStore = attachmentStore
|
||
self.backupAttachmentUploadScheduler = backupAttachmentUploadScheduler
|
||
self.backupAttachmentUploadStore = backupAttachmentUploadStore
|
||
self.logger = PrefixedLogger(prefix: "[Backups]")
|
||
self.notificationPresenter = notificationPresenter
|
||
self.orphanedBackupAttachmentStore = orphanedBackupAttachmentStore
|
||
}
|
||
|
||
func updateWithUploadedAttachment(
|
||
attachment: Attachment,
|
||
isFullsize: Bool,
|
||
remoteCdnNumber: UInt32,
|
||
tx: DBReadTransaction,
|
||
) {
|
||
// If local state says its uploaded, then all is good.
|
||
let localCdnNumber: UInt32?
|
||
let hadMediaTierInfo: Bool
|
||
if isFullsize {
|
||
hadMediaTierInfo = attachment.mediaTierInfo != nil
|
||
localCdnNumber = attachment.mediaTierInfo?.cdnNumber
|
||
} else {
|
||
hadMediaTierInfo = attachment.thumbnailMediaTierInfo != nil
|
||
localCdnNumber = attachment.thumbnailMediaTierInfo?.cdnNumber
|
||
}
|
||
|
||
switch localCdnNumber {
|
||
case nil:
|
||
if hadMediaTierInfo {
|
||
// We thought this was uploaded but didn't have the CDN number.
|
||
// This happens with attachments restored from a backup that was
|
||
// created before the attachment got uploaded by the old device;
|
||
// the attachment is optimistically treated as uploaded in the backup.
|
||
// List media discovering the cdn number in this way is expected
|
||
// and good behavior, don't mark any issues.
|
||
_result[keyPath: resultKeyPath(isFullsize: isFullsize)].uploadedCount += 1
|
||
return
|
||
} else {
|
||
// We've discovered this upload on the media tier.
|
||
let enqueuedUpload = backupAttachmentUploadStore.getEnqueuedUpload(
|
||
for: attachment.id,
|
||
fullsize: isFullsize,
|
||
tx: tx,
|
||
)
|
||
switch enqueuedUpload?.state {
|
||
case .ready:
|
||
// If it was enqueued for upload, its possible we previously attempted to upload
|
||
// and succeeded server-side but got interrupted before updating local state after,
|
||
// so its still in the upload queue. This is ok; we would have re-attempted upload
|
||
// and found it already uploaded, given the chance.
|
||
return
|
||
case .done, nil:
|
||
// If it was not in the queue, that means discovering it on the server is unexpected.
|
||
_result[keyPath: resultKeyPath(isFullsize: isFullsize)].discoveredOnCdnCount += 1
|
||
_result[keyPath: resultKeyPath(isFullsize: isFullsize)].addSampleId(attachment.id, \.discoveredOnCdnSampleAttachmentIds)
|
||
return
|
||
}
|
||
}
|
||
case remoteCdnNumber:
|
||
// Local and remote state match
|
||
_result[keyPath: resultKeyPath(isFullsize: isFullsize)].uploadedCount += 1
|
||
default:
|
||
// We thought it was uploaded, and it was, but at a different cdn number.
|
||
// This is unusual but not catastrophic; for now we only use one cdn
|
||
// number so just count it as uploaded.
|
||
_result[keyPath: resultKeyPath(isFullsize: isFullsize)].uploadedCount += 1
|
||
}
|
||
}
|
||
|
||
func updateWithUnuploadedAttachment(
|
||
attachment: Attachment,
|
||
isFullsize: Bool,
|
||
tx: DBReadTransaction,
|
||
) {
|
||
// If local state says its uploaded, and its not, that's a problem.
|
||
if isFullsize {
|
||
if attachment.mediaTierInfo?.isUploaded(currentUploadEra: uploadEraAtStartOfListMedia) == true {
|
||
_result[keyPath: resultKeyPath(isFullsize: isFullsize)].missingFromCdnCount += 1
|
||
_result[keyPath: resultKeyPath(isFullsize: isFullsize)].addSampleId(attachment.id, \.missingFromCdnSampleAttachmentIds)
|
||
return
|
||
}
|
||
} else {
|
||
if attachment.thumbnailMediaTierInfo?.isUploaded(currentUploadEra: uploadEraAtStartOfListMedia) == true {
|
||
_result[keyPath: resultKeyPath(isFullsize: isFullsize)].missingFromCdnCount += 1
|
||
_result[keyPath: resultKeyPath(isFullsize: isFullsize)].addSampleId(attachment.id, \.missingFromCdnSampleAttachmentIds)
|
||
return
|
||
}
|
||
}
|
||
|
||
// Its not uploaded; do we think its eligible?
|
||
let isEligible = backupAttachmentUploadScheduler.isEligibleToUpload(
|
||
attachment,
|
||
mode: isFullsize ? .fullsize : .thumbnail,
|
||
currentUploadEra: uploadEraAtStartOfListMedia,
|
||
tx: tx,
|
||
)
|
||
if !isEligible {
|
||
// Not uploaded, not eligible. All is good in the world.
|
||
return
|
||
}
|
||
|
||
// Check if enqueued for upload.
|
||
let enqueuedUpload = backupAttachmentUploadStore.getEnqueuedUpload(
|
||
for: attachment.id,
|
||
fullsize: isFullsize,
|
||
tx: tx,
|
||
)
|
||
switch enqueuedUpload?.state {
|
||
case .ready:
|
||
// Not uploaded, but pending upload, this is fine.
|
||
return
|
||
case .done, nil:
|
||
// Not uploaded, eligible, not scheduled. Uh-oh.
|
||
_result[keyPath: resultKeyPath(isFullsize: isFullsize)].notScheduledForUploadCount =
|
||
(_result[keyPath: resultKeyPath(isFullsize: isFullsize)].notScheduledForUploadCount ?? 0) + 1
|
||
_result[keyPath: resultKeyPath(isFullsize: isFullsize)].addSampleId(attachment.id, \.notScheduledForUploadSampleAttachmentIds)
|
||
return
|
||
}
|
||
|
||
}
|
||
|
||
func updateWithOrphanedObject(
|
||
mediaId: Data,
|
||
backupKey: MediaRootBackupKey,
|
||
tx: DBReadTransaction,
|
||
) {
|
||
if uploadEraOfLastListMedia != uploadEraAtStartOfListMedia {
|
||
// If this is our first list media for this upload era, ignore orphans we see.
|
||
// It is possible that the orphan came from another device, while this device
|
||
// was unregistered or before it ever registered, and that device never got the
|
||
// chance to issue the orphan delete before its process ended.
|
||
return
|
||
}
|
||
|
||
// First try and match by mediaId.
|
||
if orphanedBackupAttachmentStore.hasPendingDelete(forMediaId: mediaId, tx: tx) {
|
||
// Its an orphan, but one we know about. skip.
|
||
return
|
||
}
|
||
// Now check mediaNames we have pending delete, map them to mediaId, and try to match.
|
||
var foundMatch = false
|
||
orphanedBackupAttachmentStore.enumerateMediaNamesPendingDelete(tx: tx) { mediaName, stop in
|
||
let foundMediaId = try? backupKey.deriveMediaId(mediaName)
|
||
if foundMediaId == mediaId {
|
||
foundMatch = true
|
||
stop = true
|
||
}
|
||
}
|
||
if foundMatch {
|
||
// Its an orphan, but one we know about. skip.
|
||
return
|
||
}
|
||
_result.orphanedObjectCount += 1
|
||
}
|
||
|
||
func logAndNotifyIfNeeded() {
|
||
var shouldNotify = false
|
||
|
||
logger.info("\(_result.fullsize.uploadedCount) fullsize uploads")
|
||
logger.info("\(_result.fullsize.ineligibleCount) ineligible attachments")
|
||
logger.info("\(_result.thumbnail.uploadedCount) thumbnail uploads")
|
||
logger.info("\(_result.thumbnail.ineligibleCount) ineligible attachments")
|
||
if _result.fullsize.missingFromCdnCount > 0 {
|
||
shouldNotify = true
|
||
logger.error("Missing fullsize uploads from CDN, samples: \(_result.fullsize.missingFromCdnSampleAttachmentIds ?? Set())")
|
||
}
|
||
if (_result.fullsize.notScheduledForUploadCount ?? 0) > 0 {
|
||
shouldNotify = true
|
||
logger.error("Unscheduled fullsize uploads, samples: \(_result.fullsize.notScheduledForUploadSampleAttachmentIds ?? Set())")
|
||
}
|
||
if _result.fullsize.discoveredOnCdnCount > 0 {
|
||
shouldNotify = true
|
||
logger.error("Discovered fullsize upload on CDN, samples: \(_result.fullsize.discoveredOnCdnSampleAttachmentIds ?? Set())")
|
||
}
|
||
|
||
// Don't notify for thumbnail issues.
|
||
if _result.thumbnail.missingFromCdnCount > 0 {
|
||
logger.warn("Missing thumbnail uploads from CDN, samples: \(_result.thumbnail.missingFromCdnSampleAttachmentIds ?? Set())")
|
||
}
|
||
if (_result.thumbnail.notScheduledForUploadCount ?? 0) > 0 {
|
||
logger.warn("Unscheduled thumbnail uploads, samples: \(_result.thumbnail.notScheduledForUploadSampleAttachmentIds ?? Set())")
|
||
}
|
||
if _result.thumbnail.discoveredOnCdnCount > 0 {
|
||
logger.warn("Discovered thumbnail upload on CDN, samples: \(_result.thumbnail.discoveredOnCdnSampleAttachmentIds ?? Set())")
|
||
}
|
||
|
||
if _result.orphanedObjectCount > 0 {
|
||
shouldNotify = true
|
||
logger.error("Discovered \(_result.orphanedObjectCount) orphans on media tier")
|
||
}
|
||
|
||
if shouldNotify {
|
||
notificationPresenter.notifyUserOfBackupsMediaError()
|
||
}
|
||
}
|
||
|
||
private func resultKeyPath(isFullsize: Bool) -> WritableKeyPath<ListMediaIntegrityCheckResult, ListMediaIntegrityCheckResult.Result> {
|
||
return isFullsize ? \.fullsize : \.thumbnail
|
||
}
|
||
|
||
var result: ListMediaIntegrityCheckResult? {
|
||
return _result
|
||
}
|
||
}
|
||
|
||
private class ListMediaIntegrityCheckerStub: ListMediaIntegrityChecker {
|
||
|
||
init() {}
|
||
|
||
func updateWithUploadedAttachment(
|
||
attachment: Attachment,
|
||
isFullsize: Bool,
|
||
remoteCdnNumber: UInt32,
|
||
tx: DBReadTransaction,
|
||
) {}
|
||
|
||
func updateWithUnuploadedAttachment(
|
||
attachment: Attachment,
|
||
isFullsize: Bool,
|
||
tx: DBReadTransaction,
|
||
) {}
|
||
|
||
func updateWithOrphanedObject(
|
||
mediaId: Data,
|
||
backupKey: MediaRootBackupKey,
|
||
tx: DBReadTransaction,
|
||
) {}
|
||
|
||
func logAndNotifyIfNeeded() {}
|
||
|
||
var result: ListMediaIntegrityCheckResult? {
|
||
return nil
|
||
}
|
||
}
|