Merge branch 'mkirk/moving-repo'

// FREEBIE

Example/TSKitiOSTestApp/Podfile

@@ -21,7 +21,8 @@ post_install do |installer|
           existing_definitions = "$(inheritied)"
         end
 
-        config.build_settings['GCC_PREPROCESSOR_DEFINITIONS'] = "#{existing_definitions} SSK_BUILDING_FOR_TESTS=1"
+		config.build_settings['GCC_PREPROCESSOR_DEFINITIONS'] = "#{existing_definitions} SSK_BUILDING_FOR_TESTS=1"
+		config.build_settings['OTHER_SWIFT_FLAGS'] = ['$(inherited)', '-DSSK_BUILDING_FOR_TESTS']
       end
     end
   end

Example/TSKitiOSTestApp/Podfile.lock

@@ -111,7 +111,7 @@ EXTERNAL SOURCES:
   AxolotlKit:
     :git: https://github.com/WhisperSystems/SignalProtocolKit.git
   SignalServiceKit:
-    :path: "../../SignalServiceKit.podspec"
+    :path: ../../SignalServiceKit.podspec
   SocketRocket:
     :git: https://github.com/facebook/SocketRocket.git
 
@@ -140,6 +140,6 @@ SPEC CHECKSUMS:
   UnionFind: c33be5adb12983981d6e827ea94fc7f9e370f52d
   YapDatabase: cd911121580ff16675f65ad742a9eb0ab4d9e266
 
-PODFILE CHECKSUM: 124c05542083fefccb75f4b7afbdd839e27ff5ab
+PODFILE CHECKSUM: a0f4507b6b4e6f9da3250901b06187a67236e083
 
 COCOAPODS: 1.2.1

Example/TSKitiOSTestApp/TSKitiOSTestApp.xcodeproj/xcshareddata/xcschemes/TSKitiOSTestApp.xcscheme

@@ -28,7 +28,7 @@
             buildForAnalyzing = "YES">
             <BuildableReference
                BuildableIdentifier = "primary"
-               BlueprintIdentifier = "1D0826C97B09E58C83B3C228FBC535FA"
+               BlueprintIdentifier = "FF3F51F81980908EDE1836B76AA3A1EC"
                BuildableName = "libSignalServiceKit.a"
                BlueprintName = "SignalServiceKit"
                ReferencedContainer = "container:Pods/Pods.xcodeproj">

Gemfile.lock

@@ -2,7 +2,7 @@ GEM
   remote: https://rubygems.org/
   specs:
     CFPropertyList (2.3.5)
-    activesupport (4.2.8)
+    activesupport (4.2.9)
       i18n (~> 0.7)
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
@@ -10,7 +10,7 @@ GEM
     addressable (2.5.1)
       public_suffix (~> 2.0, >= 2.0.2)
     babosa (1.0.2)
-    claide (1.0.1)
+    claide (1.0.2)
     cocoapods (1.2.1)
       activesupport (>= 4.0.2, < 5)
       claide (>= 1.0.1, < 2.0)
@@ -46,14 +46,16 @@ GEM
     cocoapods-try (1.1.0)
     colored (1.2)
     colored2 (3.1.2)
-    commander-fastlane (4.4.4)
+    commander-fastlane (4.4.5)
       highline (~> 1.7.2)
+    declarative (0.0.9)
+    declarative-option (0.1.0)
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
-    dotenv (2.2.0)
+    dotenv (2.2.1)
     escape (0.0.4)
-    excon (0.55.0)
-    faraday (0.12.0.1)
+    excon (0.57.1)
+    faraday (0.12.1)
       multipart-post (>= 1.2, < 3)
     faraday-cookie_jar (0.0.6)
       faraday (>= 0.7.4)
@@ -61,20 +63,21 @@ GEM
     faraday_middleware (0.11.0.1)
       faraday (>= 0.7.4, < 1.0)
     fastimage (2.1.0)
-    fastlane (2.27.0)
+    fastlane (2.46.0)
+      CFPropertyList (>= 2.3, < 3.0.0)
       addressable (>= 2.3, < 3.0.0)
       babosa (>= 1.0.2, < 2.0.0)
       bundler (>= 1.12.0, < 2.0.0)
       colored
-      commander-fastlane (>= 4.4.0, < 5.0.0)
+      commander-fastlane (>= 4.4.5, < 5.0.0)
       dotenv (>= 2.1.1, < 3.0.0)
       excon (>= 0.45.0, < 1.0.0)
       faraday (~> 0.9)
       faraday-cookie_jar (~> 0.0.6)
       faraday_middleware (~> 0.9)
-      fastimage (>= 1.6)
+      fastimage (>= 2.1.0, < 3.0.0)
       gh_inspector (>= 1.0.1, < 2.0.0)
-      google-api-client (~> 0.9.2)
+      google-api-client (>= 0.12.0, < 0.13.0)
       highline (>= 1.7.2, < 2.0.0)
       json (< 3.0.0)
       mini_magick (~> 4.5.1)
@@ -95,15 +98,13 @@ GEM
     fourflusher (2.0.1)
     fuzzy_match (2.0.4)
     gh_inspector (1.0.3)
-    google-api-client (0.9.28)
-      addressable (~> 2.3)
+    google-api-client (0.12.0)
+      addressable (~> 2.5, >= 2.5.1)
       googleauth (~> 0.5)
-      httpclient (~> 2.7)
-      hurley (~> 0.1)
-      memoist (~> 0.11)
-      mime-types (>= 1.6)
-      representable (~> 2.3.0)
-      retriable (~> 2.0)
+      httpclient (>= 2.8.1, < 3.0)
+      mime-types (~> 3.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.0)
     googleauth (0.5.1)
       faraday (~> 0.9)
       jwt (~> 1.4)
@@ -116,20 +117,19 @@ GEM
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     httpclient (2.8.3)
-    hurley (0.2)
-    i18n (0.8.1)
-    json (2.0.4)
+    i18n (0.8.6)
+    json (2.1.0)
     jwt (1.5.6)
     little-plugger (1.1.4)
     logging (2.2.2)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
-    memoist (0.15.0)
+    memoist (0.16.0)
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
     mini_magick (4.5.1)
-    minitest (5.10.1)
+    minitest (5.10.2)
     molinillo (0.5.7)
     multi_json (1.12.1)
     multi_xml (0.6.0)
@@ -138,12 +138,14 @@ GEM
     nap (1.1.0)
     netrc (0.7.8)
     os (0.9.6)
-    plist (3.2.0)
+    plist (3.3.0)
     public_suffix (2.0.5)
-    representable (2.3.0)
-      uber (~> 0.0.7)
-    retriable (2.1.0)
-    rouge (1.11.1)
+    representable (3.0.4)
+      declarative (< 0.1.0)
+      declarative-option (< 0.2.0)
+      uber (< 0.2.0)
+    retriable (3.0.2)
+    rouge (2.0.7)
     ruby-macho (1.1.0)
     rubyzip (1.2.1)
     security (0.1.3)
@@ -153,26 +155,26 @@ GEM
       jwt (~> 1.5)
       multi_json (~> 1.10)
     slack-notifier (1.5.1)
-    terminal-notifier (1.7.1)
-    terminal-table (1.7.3)
-      unicode-display_width (~> 1.1.1)
+    terminal-notifier (1.8.0)
+    terminal-table (1.8.0)
+      unicode-display_width (~> 1.1, >= 1.1.1)
     thread_safe (0.3.6)
     tty-screen (0.5.0)
     tzinfo (1.2.3)
       thread_safe (~> 0.1)
-    uber (0.0.15)
+    uber (0.1.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.3)
-    unicode-display_width (1.1.3)
+    unf_ext (0.0.7.4)
+    unicode-display_width (1.3.0)
     word_wrap (1.0.0)
-    xcodeproj (1.4.4)
+    xcodeproj (1.5.0)
       CFPropertyList (~> 2.3.3)
-      claide (>= 1.0.1, < 2.0)
+      claide (>= 1.0.2, < 2.0)
       colored2 (~> 3.1)
       nanaimo (~> 0.2.3)
-    xcpretty (0.2.6)
-      rouge (~> 1.8)
+    xcpretty (0.2.8)
+      rouge (~> 2.0.7)
     xcpretty-travis-formatter (0.0.4)
       xcpretty (~> 0.2, >= 0.0.7)
 

README.md

@@ -1,3 +1,27 @@
+# SignalServiceKit has Moved
+
+Per https://github.com/WhisperSystems/Signal-iOS/pull/2341 we've moved
+the SignalServiceKit codebase into the primary Signal-iOS repository at
+https://github.com/WhisperSystems/Signal-iOS. As such, this repository
+will no longer be updated.
+
+Don't worry - we will continue to make updates to SignalServiceKit, and
+you can continue to use it in your projects as before. The only
+difference is where the code lives.
+
+If you are using Cocoapods, staying up to date is as simple as modifying
+a line in your Podfile from this:
+
+```
+-  pod 'SignalServiceKit', git: 'https://github.com/WhisperSystems/SignalServiceKit.git'
+```
+
+To this:
+
+```
++  pod 'SignalServiceKit', git: 'https://github.com/WhisperSystems/Signal-iOS.git'
+```
+
 # SignalServiceKit
 
 SignalServiceKit is an Objective-C library for communicating with the Signal

SignalServiceKit.podspec

@@ -17,15 +17,37 @@ An Objective-C library for communicating with the Signal messaging service.
 
   s.homepage         = "https://github.com/WhisperSystems/SignalServiceKit"
   s.license          = 'GPLv3'
-  s.author           = { "Frederic Jacobs" => "github@fredericjacobs.com" }
+  s.author           = { "Whisper Systems" => "ios@whispersystems.com" }
   s.source           = { :git => "https://github.com/WhisperSystems/SignalServiceKit.git", :tag => s.version.to_s }
-  s.social_media_url = 'https://twitter.com/FredericJacobs'
+  s.social_media_url = 'https://twitter.com/WhipserSystems'
+
+  deprecation_message = <<EOS
+installing SignalServiceKit via the Signal-iOS repository.
+
+To get future updates, point your Podfile at the new location. Simply change this:
+
+    pod 'SignalServiceKit', git: 'https://github.com/WhisperSystems/SignalServiceKit.git'
+
+To this:
+
+    pod 'SignalServiceKit', git: 'https://github.com/WhisperSystems/Signal-iOS.git'
+
+Sorry for the disruption!
+
+EOS
+
+  s.deprecated_in_favor_of = deprecation_message
 
   s.platform     = :ios, '8.0'
   #s.ios.deployment_target = '8.0'
   #s.osx.deployment_target = '10.9'
   s.requires_arc = true
-  s.source_files = 'src/**/*.{h,m,mm}'
+
+  # By not including any actual files, upgrading users will see
+  # that they need to point upgrades to the new source at
+  # https://github.com/WhisperSystems/Signal-iOS
+  # Details in README.md
+  s.source_files = 'README.md'
 
   s.resources = ['src/Security/PinningCertificate/textsecure.cer',
                  'src/Security/PinningCertificate/GIAG2.crt']

src/Contacts/PhoneNumberUtil.m

@@ -85,6 +85,9 @@
     NSDictionary *countryCodeComponent = @{NSLocaleCountryCode : countryCode};
     NSString *identifier               = [NSLocale localeIdentifierFromComponents:countryCodeComponent];
     NSString *country                  = [NSLocale.currentLocale displayNameForKey:NSLocaleIdentifier value:identifier];
+    if (country.length < 1) {
+        country = [NSLocale.systemLocale displayNameForKey:NSLocaleIdentifier value:identifier];
+    }
     return country;
 }
 

src/Messages/Attachments/TSAttachmentStream.h

@@ -46,7 +46,6 @@ NS_ASSUME_NONNULL_BEGIN
 
 + (void)deleteAttachments;
 + (NSString *)attachmentsFolder;
-+ (NSUInteger)numberOfItemsInAttachmentsFolder;
 
 - (CGSize)imageSizeWithTransaction:(YapDatabaseReadWriteTransaction *)transaction;
 - (CGSize)imageSizeWithoutTransaction;

src/Messages/Attachments/TSAttachmentStream.m

@@ -189,19 +189,6 @@ NS_ASSUME_NONNULL_BEGIN
     return attachmentsFolder;
 }
 
-+ (NSUInteger)numberOfItemsInAttachmentsFolder
-{
-    NSError *error;
-    NSUInteger count =
-        [[[NSFileManager defaultManager] contentsOfDirectoryAtPath:[self attachmentsFolder] error:&error] count];
-
-    if (error) {
-        DDLogError(@"Unable to count attachments in attachments folder. Error: %@", error);
-    }
-
-    return count;
-}
-
 - (nullable NSString *)filePath
 {
     if (!self.localRelativeFilePath) {

src/Network/API/TSNetworkManager.h

@@ -30,7 +30,7 @@ extern NSString *const TSNetworkManagerDomain;
 
 - (instancetype)init NS_UNAVAILABLE;
 
-+ (id)sharedManager;
++ (instancetype)sharedManager;
 
 - (void)makeRequest:(TSRequest *)request
             success:(void (^)(NSURLSessionDataTask *task, id responseObject))success

src/Network/API/TSNetworkManager.m

@@ -22,7 +22,8 @@ typedef void (^failureBlock)(NSURLSessionDataTask *task, NSError *error);
 
 #pragma mark Singleton implementation
 
-+ (id)sharedManager {
++ (instancetype)sharedManager
+{
     static TSNetworkManager *sharedMyManager = nil;
     static dispatch_once_t onceToken;
     dispatch_once(&onceToken, ^{

src/Storage/OWSOrphanedDataCleaner.h

@@ -1,10 +1,30 @@
-// Copyright (c) 2016 Open Whisper Systems. All rights reserved.
+//
+//  Copyright (c) 2017 Open Whisper Systems. All rights reserved.
+//
 
+NS_ASSUME_NONNULL_BEGIN
+
+// Notes:
+//
+// * On disk, we only bother cleaning up files, not directories.
+// * For code simplicity, we don't guarantee that everything is
+//   cleaned up in a single pass. If an interaction is cleaned up,
+//   it's attachments might not be cleaned up until the next pass.
+//   If an attachment is cleaned up, it's file on disk might not
+//   be cleaned up until the next pass.
 @interface OWSOrphanedDataCleaner : NSObject
 
-/**
- * Remove any inaccessible data left behind due to application bugs.
- */
-- (void)removeOrphanedData;
+- (instancetype)init NS_UNAVAILABLE;
+
++ (void)auditAsync;
+
+// completion, if present, will be invoked on the main thread.
++ (void)auditAndCleanupAsync:(void (^_Nullable)())completion;
+
++ (NSSet<NSString *> *)filePathsInAttachmentsFolder;
+
++ (long long)fileSizeOfFilePaths:(NSArray<NSString *> *)filePaths;
 
 @end
+
+NS_ASSUME_NONNULL_END

src/Storage/OWSOrphanedDataCleaner.m

@@ -9,84 +9,256 @@
 #import "TSStorageManager.h"
 #import "TSThread.h"
 
+NS_ASSUME_NONNULL_BEGIN
+
+#ifdef SSK_BUILDING_FOR_TESTS
+#define CleanupLogDebug NSLog
+#define CleanupLogInfo NSLog
+#else
+#define CleanupLogDebug DDLogDebug
+#define CleanupLogInfo DDLogInfo
+#endif
+
 @implementation OWSOrphanedDataCleaner
 
-- (void)removeOrphanedData
++ (void)auditAsync
 {
-    // Remove interactions whose threads have been deleted
-    for (NSString *interactionId in [self orphanedInteractionIds]) {
-        DDLogWarn(@"Removing orphaned interaction with id: %@", interactionId);
-        TSInteraction *interaction = [TSInteraction fetchObjectWithUniqueID:interactionId];
-        [interaction remove];
-    }
-
-    // Remove any lingering attachments
-    for (NSString *path in [self orphanedFilePaths]) {
-        DDLogWarn(@"Removing orphaned file attachment at path: %@", path);
-        NSError *error;
-        [[NSFileManager defaultManager] removeItemAtPath:path error:&error];
-        if (error) {
-            DDLogError(@"Unable to remove orphaned file attachment at path:%@", path);
-        }
-    }
+    dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+        [OWSOrphanedDataCleaner auditAndCleanup:NO completion:nil];
+    });
 }
 
-- (NSArray<NSString *> *)orphanedInteractionIds
++ (void)auditAndCleanupAsync:(void (^_Nullable)())completion
 {
-    NSMutableArray *interactionIds = [NSMutableArray new];
-    [[TSInteraction dbReadConnection] readWithBlock:^(YapDatabaseReadTransaction *_Nonnull transaction) {
-        [TSInteraction enumerateCollectionObjectsWithTransaction:transaction
-                                                      usingBlock:^(TSInteraction *interaction, BOOL *stop) {
-                                                          TSThread *thread = [TSThread
-                                                              fetchObjectWithUniqueID:interaction.uniqueThreadId
-                                                                          transaction:transaction];
-                                                          if (!thread) {
-                                                              [interactionIds addObject:interaction.uniqueId];
-                                                          }
-                                                      }];
+    dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+        [OWSOrphanedDataCleaner auditAndCleanup:YES completion:completion];
+    });
+}
+
+// This method finds and optionally cleans up:
+//
+// * Orphan messages (with no thread).
+// * Orphan attachments (with no message).
+// * Orphan attachment files (with no attachment).
+// * Missing attachment files (cannot be cleaned up).
+//   These are attachments which have no file on disk.  They should be extremely rare -
+//   the only cases I have seen are probably due to debugging.
+//   They can't be cleaned up - we don't want to delete the TSAttachmentStream or
+//   its corresponding message.  Better that the broken message shows up in the
+//   conversation view.
++ (void)auditAndCleanup:(BOOL)shouldCleanup completion:(void (^_Nullable)())completion
+{
+    NSSet<NSString *> *diskFilePaths = [self filePathsInAttachmentsFolder];
+    long long totalFileSize = [self fileSizeOfFilePaths:diskFilePaths.allObjects];
+    NSUInteger fileCount = diskFilePaths.count;
+
+    TSStorageManager *storageManager = [TSStorageManager sharedManager];
+    YapDatabaseConnection *databaseConnection = storageManager.newDatabaseConnection;
+
+    __block int attachmentStreamCount = 0;
+    NSMutableSet<NSString *> *attachmentFilePaths = [NSMutableSet new];
+    NSMutableSet<NSString *> *attachmentIds = [NSMutableSet new];
+    [databaseConnection readWithBlock:^(YapDatabaseReadTransaction *_Nonnull transaction) {
+        [transaction enumerateKeysAndObjectsInCollection:TSAttachmentStream.collection
+                                              usingBlock:^(NSString *key, TSAttachment *attachment, BOOL *stop) {
+                                                  [attachmentIds addObject:attachment.uniqueId];
+                                                  if (![attachment isKindOfClass:[TSAttachmentStream class]]) {
+                                                      return;
+                                                  }
+                                                  TSAttachmentStream *attachmentStream
+                                                      = (TSAttachmentStream *)attachment;
+                                                  attachmentStreamCount++;
+                                                  NSString *_Nullable filePath = [attachmentStream filePath];
+                                                  OWSAssert(filePath);
+                                                  [attachmentFilePaths addObject:filePath];
+                                              }];
     }];
 
+    CleanupLogDebug(@"fileCount: %zd", fileCount);
+    CleanupLogDebug(@"totalFileSize: %lld", totalFileSize);
+    CleanupLogDebug(@"attachmentStreams: %d", attachmentStreamCount);
+    CleanupLogDebug(@"attachmentStreams with file paths: %zd", attachmentFilePaths.count);
 
-    return [interactionIds copy];
+    NSMutableSet<NSString *> *orphanDiskFilePaths = [diskFilePaths mutableCopy];
+    [orphanDiskFilePaths minusSet:attachmentFilePaths];
+    NSMutableSet<NSString *> *missingAttachmentFilePaths = [attachmentFilePaths mutableCopy];
+    [missingAttachmentFilePaths minusSet:diskFilePaths];
+
+    CleanupLogDebug(@"orphan disk file paths: %zd", orphanDiskFilePaths.count);
+    CleanupLogDebug(@"missing attachment file paths: %zd", missingAttachmentFilePaths.count);
+
+    [self printPaths:orphanDiskFilePaths.allObjects label:@"orphan disk file paths"];
+    [self printPaths:missingAttachmentFilePaths.allObjects label:@"missing attachment file paths"];
+
+    NSMutableSet *threadIds = [NSMutableSet new];
+    [databaseConnection readWithBlock:^(YapDatabaseReadTransaction *_Nonnull transaction) {
+        [transaction enumerateKeysInCollection:TSThread.collection
+                                    usingBlock:^(NSString *_Nonnull key, BOOL *_Nonnull stop) {
+                                        [threadIds addObject:key];
+                                    }];
+    }];
+
+    NSMutableSet<NSString *> *orphanInteractionIds = [NSMutableSet new];
+    NSMutableSet<NSString *> *messageAttachmentIds = [NSMutableSet new];
+    [databaseConnection readWithBlock:^(YapDatabaseReadTransaction *_Nonnull transaction) {
+        [transaction enumerateKeysAndObjectsInCollection:TSMessage.collection
+                                              usingBlock:^(NSString *key, TSInteraction *interaction, BOOL *stop) {
+                                                  if (![threadIds containsObject:interaction.uniqueThreadId]) {
+                                                      [orphanInteractionIds addObject:interaction.uniqueId];
+                                                  }
+
+                                                  if (![interaction isKindOfClass:[TSMessage class]]) {
+                                                      return;
+                                                  }
+                                                  TSMessage *message = (TSMessage *)interaction;
+                                                  if (message.attachmentIds.count > 0) {
+                                                      [messageAttachmentIds addObjectsFromArray:message.attachmentIds];
+                                                  }
+                                              }];
+    }];
+
+    CleanupLogDebug(@"attachmentIds: %zd", attachmentIds.count);
+    CleanupLogDebug(@"messageAttachmentIds: %zd", messageAttachmentIds.count);
+
+    NSMutableSet<NSString *> *orphanAttachmentIds = [attachmentIds mutableCopy];
+    [orphanAttachmentIds minusSet:messageAttachmentIds];
+    NSMutableSet<NSString *> *missingAttachmentIds = [messageAttachmentIds mutableCopy];
+    [missingAttachmentIds minusSet:attachmentIds];
+
+    CleanupLogDebug(@"orphan attachmentIds: %zd", orphanAttachmentIds.count);
+    CleanupLogDebug(@"missing attachmentIds: %zd", missingAttachmentIds.count);
+    CleanupLogDebug(@"orphan interactions: %zd", orphanInteractionIds.count);
+
+    // We need to avoid cleaning up new attachments and files that are still in the process of
+    // being created/written, so we don't clean up anything recent.
+#ifdef SSK_BUILDING_FOR_TESTS
+    const NSTimeInterval kMinimumOrphanAge = 0.f;
+#else
+    const NSTimeInterval kMinimumOrphanAge = 15 * 60.f;
+#endif
+
+    if (!shouldCleanup) {
+        return;
+    }
+
+    [databaseConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *_Nonnull transaction) {
+        for (NSString *interactionId in orphanInteractionIds) {
+            TSInteraction *interaction = [TSInteraction fetchObjectWithUniqueID:interactionId transaction:transaction];
+            if (!interaction) {
+                // This could just be a race condition, but it should be very unlikely.
+                OWSFail(@"Could not load interaction: %@", interactionId);
+                continue;
+            }
+            CleanupLogInfo(@"Removing orphan message: %@", interaction.uniqueId);
+            [interaction removeWithTransaction:transaction];
+        }
+        for (NSString *attachmentId in orphanAttachmentIds) {
+            TSAttachment *attachment = [TSAttachment fetchObjectWithUniqueID:attachmentId transaction:transaction];
+            if (!attachment) {
+                // This could just be a race condition, but it should be very unlikely.
+                OWSFail(@"Could not load attachment: %@", attachmentId);
+                continue;
+            }
+            if (![attachment isKindOfClass:[TSAttachmentStream class]]) {
+                continue;
+            }
+            TSAttachmentStream *attachmentStream = (TSAttachmentStream *)attachment;
+            // Don't delete attachments which were created in the last N minutes.
+            if (fabs([attachmentStream.creationTimestamp timeIntervalSinceNow]) < kMinimumOrphanAge) {
+                CleanupLogInfo(@"Skipping orphan attachment due to age: %f",
+                    fabs([attachmentStream.creationTimestamp timeIntervalSinceNow]));
+                continue;
+            }
+            CleanupLogInfo(@"Removing orphan attachment: %@", attachmentStream.uniqueId);
+            [attachmentStream removeWithTransaction:transaction];
+        }
+    }];
+
+    for (NSString *filePath in orphanDiskFilePaths) {
+        NSError *error;
+        NSDictionary *attributes = [[NSFileManager defaultManager] attributesOfItemAtPath:filePath error:&error];
+        if (!attributes || error) {
+            OWSFail(@"Could not get attributes of file at: %@", filePath);
+            continue;
+        }
+        // Don't delete files which were created in the last N minutes.
+        if (fabs([attributes.fileModificationDate timeIntervalSinceNow]) < kMinimumOrphanAge) {
+            CleanupLogInfo(@"Skipping orphan attachment file due to age: %f",
+                fabs([attributes.fileModificationDate timeIntervalSinceNow]));
+            continue;
+        }
+
+        CleanupLogInfo(@"Removing orphan attachment file: %@", filePath);
+        [[NSFileManager defaultManager] removeItemAtPath:filePath error:&error];
+        if (error) {
+            OWSFail(@"Could not remove orphan file at: %@", filePath);
+        }
+    }
+
+    if (completion) {
+        dispatch_async(dispatch_get_main_queue(), ^{
+            completion();
+        });
+    }
 }
 
-- (NSArray<NSString *> *)orphanedFilePaths
++ (void)printPaths:(NSArray<NSString *> *)paths label:(NSString *)label
+{
+    for (NSString *path in [paths sortedArrayUsingSelector:@selector(compare:)]) {
+        CleanupLogDebug(@"%@: %@", label, path);
+    }
+}
+
++ (NSSet<NSString *> *)filePathsInAttachmentsFolder
+{
+    NSString *attachmentsFolder = [TSAttachmentStream attachmentsFolder];
+    CleanupLogDebug(@"attachmentsFolder: %@", attachmentsFolder);
+
+    return [self filePathsInDirectory:attachmentsFolder];
+}
+
++ (NSSet<NSString *> *)filePathsInDirectory:(NSString *)dirPath
+{
+    NSMutableSet *filePaths = [NSMutableSet new];
+    NSError *error;
+    NSArray<NSString *> *fileNames = [[NSFileManager defaultManager] contentsOfDirectoryAtPath:dirPath error:&error];
+    if (error) {
+        OWSFail(@"contentsOfDirectoryAtPath error: %@", error);
+        return [NSSet new];
+    }
+    for (NSString *fileName in fileNames) {
+        NSString *filePath = [dirPath stringByAppendingPathComponent:fileName];
+        BOOL isDirectory;
+        [[NSFileManager defaultManager] fileExistsAtPath:filePath isDirectory:&isDirectory];
+        if (isDirectory) {
+            [filePaths addObjectsFromArray:[self filePathsInDirectory:filePath].allObjects];
+        } else {
+            [filePaths addObject:filePath];
+        }
+    }
+    return filePaths;
+}
+
++ (long long)fileSizeOfFilePath:(NSString *)filePath
 {
     NSError *error;
-    NSMutableArray<NSString *> *filenames =
-        [[[NSFileManager defaultManager] contentsOfDirectoryAtPath:[TSAttachmentStream attachmentsFolder] error:&error]
-            mutableCopy];
+    NSNumber *fileSize = [[NSFileManager defaultManager] attributesOfItemAtPath:filePath error:&error][NSFileSize];
     if (error) {
-        DDLogError(@"error getting orphanedFilePaths:%@", error);
-        return @[];
+        OWSFail(@"attributesOfItemAtPath: %@ error: %@", filePath, error);
+        return 0;
     }
+    return fileSize.longLongValue;
+}
 
-    NSMutableDictionary<NSString *, NSString *> *attachmentIdFilenames = [NSMutableDictionary new];
-    for (NSString *filename in filenames) {
-        // Remove extension from (e.g.) 1234.png to get the attachmentId "1234"
-        NSString *attachmentId = [filename stringByDeletingPathExtension];
-        attachmentIdFilenames[attachmentId] = filename;
++ (long long)fileSizeOfFilePaths:(NSArray<NSString *> *)filePaths
+{
+    long long result = 0;
+    for (NSString *filePath in filePaths) {
+        result += [self fileSizeOfFilePath:filePath];
     }
-
-    [TSInteraction enumerateCollectionObjectsUsingBlock:^(TSInteraction *interaction, BOOL *stop) {
-        if ([interaction isKindOfClass:[TSMessage class]]) {
-            TSMessage *message = (TSMessage *)interaction;
-            if ([message hasAttachments]) {
-                for (NSString *attachmentId in message.attachmentIds) {
-                    [attachmentIdFilenames removeObjectForKey:attachmentId];
-                }
-            }
-        }
-    }];
-
-    NSArray<NSString *> *filenamesToDelete = [attachmentIdFilenames allValues];
-    NSMutableArray<NSString *> *absolutePathsToDelete = [NSMutableArray arrayWithCapacity:[filenamesToDelete count]];
-    for (NSString *filename in filenamesToDelete) {
-        NSString *absolutePath = [[TSAttachmentStream attachmentsFolder] stringByAppendingFormat:@"/%@", filename];
-        [absolutePathsToDelete addObject:absolutePath];
-    }
-
-    return [absolutePathsToDelete copy];
+    return result;
 }
 
 @end
+
+NS_ASSUME_NONNULL_END

src/Storage/TSDatabaseView.m

@@ -21,7 +21,9 @@ NSString *const TSArchiveGroup = @"TSArchiveGroup";
 NSString *const TSUnreadIncomingMessagesGroup = @"TSUnreadIncomingMessagesGroup";
 NSString *const TSSecondaryDevicesGroup = @"TSSecondaryDevicesGroup";
 
-NSString *const TSThreadDatabaseViewExtensionName = @"TSThreadDatabaseViewExtensionName";
+// YAPDB BUG: when changing from non-persistent to persistent view, we had to rename TSThreadDatabaseViewExtensionName
+// -> TSThreadDatabaseViewExtensionName2 to work around https://github.com/yapstudios/YapDatabase/issues/324
+NSString *const TSThreadDatabaseViewExtensionName = @"TSThreadDatabaseViewExtensionName2";
 NSString *const TSMessageDatabaseViewExtensionName = @"TSMessageDatabaseViewExtensionName";
 NSString *const TSThreadOutgoingMessageDatabaseViewExtensionName = @"TSThreadOutgoingMessageDatabaseViewExtensionName";
 NSString *const TSUnreadDatabaseViewExtensionName = @"TSUnreadDatabaseViewExtensionName";

src/Util/MIMETypeUtil.m

@@ -420,6 +420,12 @@ NSString *const OWSMimeTypeUnknownForTests = @"unknown/mimetype";
 
 + (nullable NSString *)fileExtensionForUTIType:(NSString *)utiType
 {
+    // Special-case the "aac" filetype we use for voice messages (for legacy reasons)
+    // to use a .m4a file extension, not .aac, since AVAudioPlayer can't handle .aac
+    // properly. Doesn't affect file contents.
+    if ([utiType isEqualToString:@"public.aac-audio"]) {
+        return @"m4a";
+    }
     CFStringRef fileExtension
         = UTTypeCopyPreferredTagWithClass((__bridge CFStringRef)utiType, kUTTagClassFilenameExtension);
     return (__bridge_transfer NSString *)fileExtension;

tests/Storage/OWSOrphanedDataCleanerTest.m

@@ -14,6 +14,8 @@
 
 @end
 
+#pragma mark -
+
 @implementation OWSOrphanedDataCleanerTest
 
 - (void)setUp
@@ -24,7 +26,7 @@
 
     // Set up initial conditions & Sanity check
     [TSAttachmentStream deleteAttachments];
-    XCTAssertEqual(0, [TSAttachmentStream numberOfItemsInAttachmentsFolder]);
+    XCTAssertEqual(0, [self numberOfItemsInAttachmentsFolder]);
     [TSAttachmentStream removeAllObjectsInCollection];
     XCTAssertEqual(0, [TSAttachmentStream numberOfKeysInCollection]);
     [TSIncomingMessage removeAllObjectsInCollection];
@@ -38,6 +40,11 @@
     [super tearDown];
 }
 
+- (NSUInteger)numberOfItemsInAttachmentsFolder
+{
+    return [OWSOrphanedDataCleaner filePathsInAttachmentsFolder].count;
+}
+
 - (void)testInteractionsWithoutThreadAreDeleted
 {
     // This thread is intentionally not saved. It's meant to recreate a situation we've seen where interactions exist
@@ -53,7 +60,17 @@
     [incomingMessage save];
     XCTAssertEqual(1, [TSIncomingMessage numberOfKeysInCollection]);
 
-    [[OWSOrphanedDataCleaner new] removeOrphanedData];
+    XCTestExpectation *expectation = [self expectationWithDescription:@"Cleanup"];
+    [OWSOrphanedDataCleaner auditAndCleanupAsync:^{
+        [expectation fulfill];
+    }];
+    [self waitForExpectationsWithTimeout:5.0
+                                 handler:^(NSError *error) {
+                                     if (error) {
+                                         XCTFail(@"Expectation Failed with error: %@", error);
+                                     }
+                                 }];
+
     XCTAssertEqual(0, [TSIncomingMessage numberOfKeysInCollection]);
 }
 
@@ -70,14 +87,24 @@
     [incomingMessage save];
     XCTAssertEqual(1, [TSIncomingMessage numberOfKeysInCollection]);
 
-    [[OWSOrphanedDataCleaner new] removeOrphanedData];
+    XCTestExpectation *expectation = [self expectationWithDescription:@"Cleanup"];
+    [OWSOrphanedDataCleaner auditAndCleanupAsync:^{
+        [expectation fulfill];
+    }];
+    [self waitForExpectationsWithTimeout:5.0
+                                 handler:^(NSError *error) {
+                                     if (error) {
+                                         XCTFail(@"Expectation Failed with error: %@", error);
+                                     }
+                                 }];
+
     XCTAssertEqual(1, [TSIncomingMessage numberOfKeysInCollection]);
 }
 
 - (void)testFilesWithoutInteractionsAreDeleted
 {
     // sanity check
-    XCTAssertEqual(0, [TSAttachmentStream numberOfItemsInAttachmentsFolder]);
+    XCTAssertEqual(0, [self numberOfItemsInAttachmentsFolder]);
 
     NSError *error;
     TSAttachmentStream *attachmentStream = [[TSAttachmentStream alloc] initWithContentType:@"image/jpeg" sourceFilename:nil];
@@ -86,12 +113,25 @@
     NSString *orphanedFilePath = [attachmentStream filePath];
     BOOL fileExists = [[NSFileManager defaultManager] fileExistsAtPath:orphanedFilePath];
     XCTAssert(fileExists);
-    XCTAssertEqual(1, [TSAttachmentStream numberOfItemsInAttachmentsFolder]);
+    XCTAssertEqual(1, [self numberOfItemsInAttachmentsFolder]);
+
+    // Do multiple cleanup passes.
+    for (int i = 0; i < 2; i++) {
+        XCTestExpectation *expectation = [self expectationWithDescription:@"Cleanup"];
+        [OWSOrphanedDataCleaner auditAndCleanupAsync:^{
+            [expectation fulfill];
+        }];
+        [self waitForExpectationsWithTimeout:5.0
+                                     handler:^(NSError *error) {
+                                         if (error) {
+                                             XCTFail(@"Expectation Failed with error: %@", error);
+                                         }
+                                     }];
+    }
 
-    [[OWSOrphanedDataCleaner new] removeOrphanedData];
     fileExists = [[NSFileManager defaultManager] fileExistsAtPath:orphanedFilePath];
     XCTAssertFalse(fileExists);
-    XCTAssertEqual(0, [TSAttachmentStream numberOfItemsInAttachmentsFolder]);
+    XCTAssertEqual(0, [self numberOfItemsInAttachmentsFolder]);
 }
 
 - (void)testFilesWithInteractionsAreNotDeleted
@@ -116,13 +156,22 @@
     NSString *attachmentFilePath = [attachmentStream filePath];
     BOOL fileExists = [[NSFileManager defaultManager] fileExistsAtPath:attachmentFilePath];
     XCTAssert(fileExists);
-    XCTAssertEqual(1, [TSAttachmentStream numberOfItemsInAttachmentsFolder]);
+    XCTAssertEqual(1, [self numberOfItemsInAttachmentsFolder]);
 
-    [[OWSOrphanedDataCleaner new] removeOrphanedData];
+    XCTestExpectation *expectation = [self expectationWithDescription:@"Cleanup"];
+    [OWSOrphanedDataCleaner auditAndCleanupAsync:^{
+        [expectation fulfill];
+    }];
+    [self waitForExpectationsWithTimeout:5.0
+                                 handler:^(NSError *error) {
+                                     if (error) {
+                                         XCTFail(@"Expectation Failed with error: %@", error);
+                                     }
+                                 }];
 
     fileExists = [[NSFileManager defaultManager] fileExistsAtPath:attachmentFilePath];
     XCTAssert(fileExists);
-    XCTAssertEqual(1, [TSAttachmentStream numberOfItemsInAttachmentsFolder]);
+    XCTAssertEqual(1, [self numberOfItemsInAttachmentsFolder]);
 }
 
 - (void)testFilesWithoutAttachmentStreamsAreDeleted
@@ -135,12 +184,22 @@
     NSString *orphanedFilePath = [attachmentStream filePath];
     BOOL fileExists = [[NSFileManager defaultManager] fileExistsAtPath:orphanedFilePath];
     XCTAssert(fileExists);
-    XCTAssertEqual(1, [TSAttachmentStream numberOfItemsInAttachmentsFolder]);
+    XCTAssertEqual(1, [self numberOfItemsInAttachmentsFolder]);
+
+    XCTestExpectation *expectation = [self expectationWithDescription:@"Cleanup"];
+    [OWSOrphanedDataCleaner auditAndCleanupAsync:^{
+        [expectation fulfill];
+    }];
+    [self waitForExpectationsWithTimeout:5.0
+                                 handler:^(NSError *error) {
+                                     if (error) {
+                                         XCTFail(@"Expectation Failed with error: %@", error);
+                                     }
+                                 }];
 
-    [[OWSOrphanedDataCleaner new] removeOrphanedData];
     fileExists = [[NSFileManager defaultManager] fileExistsAtPath:orphanedFilePath];
     XCTAssertFalse(fileExists);
-    XCTAssertEqual(0, [TSAttachmentStream numberOfItemsInAttachmentsFolder]);
+    XCTAssertEqual(0, [self numberOfItemsInAttachmentsFolder]);
 }
 
 @end