From 158010958aef99d5337635731027ac72c5554fa2 Mon Sep 17 00:00:00 2001 From: Harry <109690906+harry-signal@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:40:05 -0700 Subject: [PATCH] Fork relevant parts of AudioWaveformManager for TSAttachmentMigration --- Signal.xcodeproj/project.pbxproj | 4 + ...chmentMigration+AudioWaveformManager.swift | 273 ++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 SignalServiceKit/Storage/Database/IncrementalMigrations/TSAttachment/TSAttachmentMigration+AudioWaveformManager.swift diff --git a/Signal.xcodeproj/project.pbxproj b/Signal.xcodeproj/project.pbxproj index 28ea2e6ea6..94b1cc3886 100644 --- a/Signal.xcodeproj/project.pbxproj +++ b/Signal.xcodeproj/project.pbxproj @@ -1009,6 +1009,7 @@ 6691E7F22996E9BC0032A68A /* RegistrationSessionManagerMock.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6691E7F12996E9BC0032A68A /* RegistrationSessionManagerMock.swift */; }; 6691E7F72996EAD70032A68A /* SecureValueRecoveryMock.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6691E7F62996EAD70032A68A /* SecureValueRecoveryMock.swift */; }; 669379ED2C3C5B2C00EED7A0 /* TSAttachmentMigration+Records.swift in Sources */ = {isa = PBXBuildFile; fileRef = 669379EC2C3C5B2C00EED7A0 /* TSAttachmentMigration+Records.swift */; }; + 669379EF2C3C5E5800EED7A0 /* TSAttachmentMigration+AudioWaveformManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 669379EE2C3C5E5800EED7A0 /* TSAttachmentMigration+AudioWaveformManager.swift */; }; 6694BF682B36484900B18764 /* PinnedThreadManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6694BF672B36484800B18764 /* PinnedThreadManager.swift */; }; 6694BF6A2B3650E400B18764 /* PinnedThreadStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6694BF692B3650E400B18764 /* PinnedThreadStore.swift */; }; 6694BF6C2B3656D500B18764 /* MockPinnedThreadManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6694BF6B2B3656D500B18764 /* MockPinnedThreadManager.swift */; }; @@ -4022,6 +4023,7 @@ 6691E7F12996E9BC0032A68A /* RegistrationSessionManagerMock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RegistrationSessionManagerMock.swift; sourceTree = ""; }; 6691E7F62996EAD70032A68A /* SecureValueRecoveryMock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SecureValueRecoveryMock.swift; sourceTree = ""; }; 669379EC2C3C5B2C00EED7A0 /* TSAttachmentMigration+Records.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TSAttachmentMigration+Records.swift"; sourceTree = ""; }; + 669379EE2C3C5E5800EED7A0 /* TSAttachmentMigration+AudioWaveformManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TSAttachmentMigration+AudioWaveformManager.swift"; sourceTree = ""; }; 6694BF672B36484800B18764 /* PinnedThreadManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PinnedThreadManager.swift; sourceTree = ""; }; 6694BF692B3650E400B18764 /* PinnedThreadStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PinnedThreadStore.swift; sourceTree = ""; }; 6694BF6B2B3656D500B18764 /* MockPinnedThreadManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MockPinnedThreadManager.swift; sourceTree = ""; }; @@ -8175,6 +8177,7 @@ 66A1ABE02C33118A0033C5EB /* TSAttachment */ = { isa = PBXGroup; children = ( + 669379EE2C3C5E5800EED7A0 /* TSAttachmentMigration+AudioWaveformManager.swift */, 669379EC2C3C5B2C00EED7A0 /* TSAttachmentMigration+Records.swift */, 66A1ABE12C3311B40033C5EB /* TSAttachmentMigration.swift */, ); @@ -14918,6 +14921,7 @@ 669FAE152B75968D009EE2FE /* TSAttachmentDownloadJob.swift in Sources */, F9C5CC72289453B300548EEE /* TSAttachmentDownloadManager.swift in Sources */, 66E1AD962B91437B00C56B7B /* TSAttachmentManager.swift in Sources */, + 669379EF2C3C5E5800EED7A0 /* TSAttachmentMigration+AudioWaveformManager.swift in Sources */, 669379ED2C3C5B2C00EED7A0 /* TSAttachmentMigration+Records.swift in Sources */, 66A1ABE22C3311B40033C5EB /* TSAttachmentMigration.swift in Sources */, 725464EC2B9F6EB900EABFD2 /* TSAttachmentMultisendJob.swift in Sources */, diff --git a/SignalServiceKit/Storage/Database/IncrementalMigrations/TSAttachment/TSAttachmentMigration+AudioWaveformManager.swift b/SignalServiceKit/Storage/Database/IncrementalMigrations/TSAttachment/TSAttachmentMigration+AudioWaveformManager.swift new file mode 100644 index 0000000000..e043ba42fa --- /dev/null +++ b/SignalServiceKit/Storage/Database/IncrementalMigrations/TSAttachment/TSAttachmentMigration+AudioWaveformManager.swift @@ -0,0 +1,273 @@ +// +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only +// + +import Accelerate +import AVFoundation +import Foundation + +extension TSAttachmentMigration { + + struct AudioWaveform { + let decibelSamples: [Float] + + func archive() throws -> Data { + return try NSKeyedArchiver.archivedData(withRootObject: decibelSamples, requiringSecureCoding: false) + } + } + + class AudioWaveformManager { + + static func buildAudioWaveForm( + unencryptedFilePath: String + ) throws -> TSAttachmentMigration.AudioWaveform { + let asset: AVAsset = try assetFromUnencryptedAudioFile(atAudioPath: unencryptedFilePath) + + guard asset.isReadable else { + throw OWSAssertionError("unexpectedly encountered unreadable audio file.") + } + + guard CMTimeGetSeconds(asset.duration) <= Self.maximumDuration else { + throw OWSAssertionError("Audio too long") + } + + return try sampleWaveform(asset: asset) + } + + private static func assetFromUnencryptedAudioFile( + atAudioPath audioPath: String + ) throws -> AVAsset { + let audioUrl = URL(fileURLWithPath: audioPath) + + var asset = AVURLAsset(url: audioUrl) + + if !asset.isReadable { + if let extensionOverride = Self.alternativeAudioFileExtension(fileExtension: audioUrl.pathExtension) { + let symlinkPath = OWSFileSystem.temporaryFilePath( + fileExtension: extensionOverride, + isAvailableWhileDeviceLocked: true + ) + try FileManager.default.createSymbolicLink( + atPath: symlinkPath, + withDestinationPath: audioPath + ) + asset = AVURLAsset(url: URL(fileURLWithPath: symlinkPath)) + } + } + + return asset + } + + private static func alternativeAudioFileExtension(fileExtension: String) -> String? { + // In some cases, Android sends audio messages with the "audio/mpeg" mime type. This + // makes our choice of file extension ambiguous—`.mp3` or `.m4a`? AVFoundation uses the + // extension to read the file, and if the extension is wrong, it won't be readable. + // + // We "lie" about the extension to generate the waveform so that AVFoundation may read + // it. This is brittle but necessary to work around the buggy marriage of Android's + // content type and AVFoundation's behavior. + // + // Note that we probably still want this code even if Android updates theirs, because + // iOS users might have existing attachments. + // + // See: + // . + switch fileExtension { + case "m4a": return "aac" + case "mp3": return "m4a" + default: return nil + } + } + + // MARK: - Sampling + + /// The maximum duration asset that we will display waveforms for. + /// It's too intensive to sample a waveform for really long audio files. + private static let maximumDuration: TimeInterval = 15 * kMinuteInterval + private static let silenceThreshold: Float = -50 + private static let sampleCount = 100 + + private static func downsample(samples: [Float], toSampleCount sampleCount: Int) -> [Float] { + // Do nothing if the number of requested samples is less than 1 + guard sampleCount > 0 else { return [] } + + // If the requested sample count is equal to the sample count, just return the samples + guard samples.count != sampleCount else { return samples } + + // Calculate the number of samples each downsampled value should take into account + let sampleDistribution = Float(samples.count) / Float(sampleCount) + + // Calculate the number of samples we need to factor in when downsampling. Since there + // is no such thing as a fractional sample, we need to round this up to the nearest Int. + // When we calculated the distribution later, it will factor in that some of these samples + // are weighted differently than others in the resulting output. + let sampleLength = Int(ceil(sampleDistribution)) + + // Calculate the weight of each sample in the downsampled group. + // For whole number `sampleDistribution` the distribution is always + // equivalent across all of the samples. If the sampleDistribution + // is _not_ a whole number, we factor the bookending values in + // relative to remainder proportion + let distribution: [Float] = { + let averageProportion = 1 / sampleDistribution + + var array = [Float](repeating: averageProportion, count: sampleLength) + + if samples.count % sampleCount != 0 { + // Calculate the proportion that the partial sample should be weighted at + let remainderProportion = (sampleDistribution.truncatingRemainder(dividingBy: 1)) * averageProportion + + // The partial sample is factored into the "bookends" (first and last element of the distribution) + // by averaging the average distribution and the remainder distribution together. + // This provides a lightweight "anti-aliasing" effect. + let bookEndProportions = (averageProportion + remainderProportion) / 2 + + array[0] = bookEndProportions + array[sampleLength - 1] = bookEndProportions + } + + return array + }() + + // If we can ever guarantee that `samples.count` is always a multiple of `sampleCount`, we should + // switch to using the faster `vDSP_desamp`. For now, we can't use it since it only supports the + // integer stride lengths. This should be okay, since this should only be operating on already + // downsampled data (~100 points) rather than the original millions of points. + let result: [Float] = (0.. TSAttachmentMigration.AudioWaveform { + let assetReader = try AVAssetReader(asset: asset) + + // We just draw the waveform based on the first track. + guard let audioTrack = assetReader.asset.tracks.first else { + throw OWSAssertionError("audio file has no tracks") + } + + let trackOutput = AVAssetReaderTrackOutput( + track: audioTrack, + outputSettings: [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsBigEndianKey: false, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsNonInterleaved: false + ] + ) + assetReader.add(trackOutput) + + let decibelSamples = try readDecibels(from: assetReader) + + return TSAttachmentMigration.AudioWaveform(decibelSamples: decibelSamples) + } + + private static func readDecibels(from assetReader: AVAssetReader) throws -> [Float] { + var outputSamples = [Float]() + var readBuffer = Data() + + let samplesToGroup = max(1, sampleCount(from: assetReader) / Self.sampleCount) + + assetReader.startReading() + while assetReader.status == .reading { + guard let trackOutput = assetReader.outputs.first else { + throw OWSAssertionError("track output unexpectedly missing") + } + + // Process any newly read data. + guard let nextSampleBuffer = trackOutput.copyNextSampleBuffer(), + let blockBuffer = CMSampleBufferGetDataBuffer(nextSampleBuffer) else { + // There is no more data to read, break + break + } + + var readBufferLength = 0 + var readBufferPointer: UnsafeMutablePointer? + CMBlockBufferGetDataPointer( + blockBuffer, + atOffset: 0, + lengthAtOffsetOut: &readBufferLength, + totalLengthOut: nil, + dataPointerOut: &readBufferPointer + ) + readBuffer.append(UnsafeBufferPointer(start: readBufferPointer, count: readBufferLength)) + CMSampleBufferInvalidate(nextSampleBuffer) + + // Try and process any pending samples, we may not have read enough data yet to do this. + let processedSamples = convertToDecibels(fromAmplitudes: readBuffer, from: assetReader, groupSize: samplesToGroup) + outputSamples += processedSamples + + // If we successfully processed samples, remove any processed samples + // from the read buffer. + if processedSamples.count > 0 { + readBuffer.removeFirst(processedSamples.count * samplesToGroup * MemoryLayout.size) + } + } + + return outputSamples + } + + private static func sampleCount(from assetReader: AVAssetReader) -> Int { + let samplesPerChannel = Int(assetReader.asset.duration.value) + + // We will read in the samples from each channel, interleaved since + // we only draw one waveform. This gives us an average of the channels + // if it is, for example, a stereo audio file. + return samplesPerChannel * channelCount(from: assetReader) + } + + private static func channelCount(from assetReader: AVAssetReader) -> Int { + guard let output = assetReader.outputs.first as? AVAssetReaderTrackOutput, + let formatDescriptions = output.track.formatDescriptions as? [CMFormatDescription] else { return 0 } + + var channelCount = 0 + + for description in formatDescriptions { + guard let basicDescription = CMAudioFormatDescriptionGetStreamBasicDescription(description) else { continue } + channelCount = Int(basicDescription.pointee.mChannelsPerFrame) + } + + return channelCount + } + + private static func convertToDecibels(fromAmplitudes sampleBuffer: Data, from assetReader: AVAssetReader, groupSize: Int) -> [Float] { + var downSampledData = [Float]() + + sampleBuffer.withUnsafeBytes { samples in + let sampleLength = sampleBuffer.count / MemoryLayout.size + var decibelSamples = [Float](repeating: 0.0, count: sampleLength) + + // maximum amplitude storable in Int16 = 0 dB (loudest) + var zeroDecibelEquivalent: Float = Float(Int16.max) + + var loudestClipValue: Float = 0.0 + var quietestClipValue = Self.silenceThreshold + let samplesToProcess = vDSP_Length(sampleLength) + + // convert 16bit int amplitudes to float representation + vDSP_vflt16([Int16](samples.bindMemory(to: Int16.self)), 1, &decibelSamples, 1, samplesToProcess) + + // take the absolute amplitude value + vDSP_vabs(decibelSamples, 1, &decibelSamples, 1, samplesToProcess) + + // convert to dB + vDSP_vdbcon(decibelSamples, 1, &zeroDecibelEquivalent, &decibelSamples, 1, samplesToProcess, 1) + + // clip between loudest + quietest + vDSP_vclip(decibelSamples, 1, &quietestClipValue, &loudestClipValue, &decibelSamples, 1, samplesToProcess) + + let sampleCount = sampleLength / groupSize + downSampledData = downsample(samples: decibelSamples, toSampleCount: sampleCount) + } + + return downSampledData + } + } +}