Merge branch 'mkirk/smaller-batches' into signal-release

YapDatabase/Extensions/View/Utilities/YapDatabaseViewChange.m

@@ -2153,24 +2153,30 @@
 		// 1 -> 2
 		// 2 -> 1
 		// 3 -> 0
-		//
-		// Basically, we find the midpoint, and then move each index to the other side of the midpoint,
-		// but we keep its distance from the midpoint the same.
-			
 		if ([reverse containsObject:rowChange->originalGroup])
 		{
 			NSUInteger count = [originalMappings visibleCountForGroup:rowChange->originalGroup];
-			double mid = (count - 1) / 2.0;
-			
-			rowChange->originalIndex = (NSUInteger)(mid - (rowChange->originalIndex - mid));
+			NSUInteger forwardIndex = rowChange->originalIndex;
+			if (count < forwardIndex + 1) {
+				// Calculating the reverse index would overflow, so we skip it.
+				NSAssert(NO, @"original index is too large");
+			} else {
+				rowChange->originalIndex = count - (forwardIndex + 1);
+			}
 		}
 		
 		if ([reverse containsObject:rowChange->finalGroup])
 		{
 			NSUInteger count = [finalMappings visibleCountForGroup:rowChange->finalGroup];
-			double mid = (count - 1) / 2.0;
-			
-			rowChange->finalIndex = (NSUInteger)(mid - (rowChange->finalIndex - mid));
+			NSUInteger forwardIndex = rowChange->finalIndex;
+			if (count < forwardIndex + 1) {
+				// This can happen when deleting an item.
+				// Calculating the reverse index would overflow, so we skip it.
+				// When deleting a row, the finalIndex is irrelevant anyway.
+				NSAssert(rowChange->type == YapDatabaseViewChangeDelete, @"final index is too large");
+			} else {
+				rowChange->finalIndex = count - (forwardIndex + 1);
+			}
 		}
 	}
 	

YapDatabase/Extensions/View/YapDatabaseViewTransaction.m

@@ -2980,18 +2980,19 @@
 	
 	for (YapDatabaseViewPageMetadata *pageMetadata in pagesMetadataForGroup)
 	{
-		YapDatabaseViewPage *page = [self pageForPageKey:pageMetadata->pageKey];
-		
-		[page enumerateRowidsUsingBlock:^(int64_t rowid, NSUInteger idx, BOOL *innerStop) {
-			
-			block(rowid, pageOffset+idx, &stop);
-			
-			if (stop || [self->parentConnection->mutatedGroups containsObject:group]) *innerStop = YES;
-		}];
-		
-		if (stop || [parentConnection->mutatedGroups containsObject:group]) break;
-		
-		pageOffset += pageMetadata->count;
+        @autoreleasepool {
+            YapDatabaseViewPage *page = [self pageForPageKey:pageMetadata->pageKey];
+            [page enumerateRowidsUsingBlock:^(int64_t rowid, NSUInteger idx, BOOL *innerStop) {
+                
+                block(rowid, pageOffset+idx, &stop);
+                
+                if (stop || [self->parentConnection->mutatedGroups containsObject:group]) *innerStop = YES;
+            }];
+            
+            if (stop || [parentConnection->mutatedGroups containsObject:group]) break;
+            
+            pageOffset += pageMetadata->count;
+        }
 	}
 	
 	if (!stop && [parentConnection->mutatedGroups containsObject:group])

YapDatabase/Utilities/YapDatabaseCryptoUtils.h

@@ -6,6 +6,8 @@
 
 NS_ASSUME_NONNULL_BEGIN
 
+@class YapDatabaseOptions;
+
 #ifdef SQLITE_HAS_CODEC
 
 extern const NSUInteger kSqliteHeaderLength;
@@ -13,8 +15,12 @@ extern const NSUInteger kSQLCipherSaltLength;
 extern const NSUInteger kSQLCipherDerivedKeyLength;
 extern const NSUInteger kSQLCipherKeySpecLength;
 
-typedef void (^YapDatabaseSaltBlock)(NSData *saltData);
-typedef void (^YapDatabaseKeySpecBlock)(NSData *keySpecData);
+// User specified block used to notify the caller of a database's salt when
+// converting SQLCipher headers to plaintext. Failing to properly record the
+// salt will leave the database unreadable.
+// @returns BOOL indicating if the salt was successfully recorded. Conversion
+//          will not proceed if recording the salt fails.
+typedef BOOL (^YapRecordDatabaseSaltBlock)(NSData *saltData);
 
 // This class contains utility methods for use with SQLCipher encrypted
 // databases, specifically to address an issue around database files that
@@ -58,13 +64,14 @@ typedef void (^YapDatabaseKeySpecBlock)(NSData *keySpecData);
 // The header does not contain any user data.  See:
 // https://www.sqlite.org/fileformat.html#the_database_header
 //
-// However, Sqlite normally uses the first 16 bytes of the Sqlite header to store
+// However, SQLCipher normally uses the first 16 bytes of the Sqlite header to store
 // a salt value.  Therefore when using unencrypted headers, it is also necessary
 // to explicitly specify a salt value.
 //
 // It is possible to convert SQLCipher databases with encrypted headers to use
 // unencrypted headers.  However, during this conversion, the salt must be extracted
-// and preserved by reading the first 16 bytes of the unconverted file.
+// by reading the first 16 bytes of the unconverted file and preserving it elsewhere,
+// e.g. the keychain.
 //
 //
 // Implementation
@@ -99,9 +106,9 @@ typedef void (^YapDatabaseKeySpecBlock)(NSData *keySpecData);
 // * This method should always be pretty fast, and should be safe to
 //   call from within [UIApplicationDelegate application: didFinishLaunchingWithOptions:].
 // * If convertDatabaseIfNecessary converts the database, it will use its
-//   saltBlock and keySpecBlock parameters to inform you of the salt
+//   recordSaltBlock to inform you of the salt
 //   and keyspec for this database.  These values will be needed when
-//   opening the database, so they should presumably stored in the
+//   opening the database, so they should presumably be stored in the
 //   keychain (like the database password).
 //
 //
@@ -130,23 +137,22 @@ typedef void (^YapDatabaseKeySpecBlock)(NSData *keySpecData);
 // * This method will have no effect if the YapDatabase has already been converted.
 // * This method should always be pretty fast, and should be safe to
 //   call from within [UIApplicationDelegate application: didFinishLaunchingWithOptions:].
-// * If convertDatabaseIfNecessary converts the database, it will use its
-//   saltBlock and keySpecBlock parameters to inform you of the salt
-//   and keyspec for this database.  These values will be needed when
-//   opening the database, so they should presumably stored in the
-//   keychain (like the database password).
+// * IMPORTANT: If you fail to record the salt during conversion you will not be able to decrypt
+//   the database in the future, effectively losing all data.  If convertDatabaseIfNecessary
+//   converts the database, it will use its recordSaltBlock parameter to inform you of the salt
+//   for this database. Within that block you must store the salt somewhere durable.
 + (nullable NSError *)convertDatabaseIfNecessary:(NSString *)databaseFilePath
                                 databasePassword:(NSData *)databasePassword
-                                       saltBlock:(YapDatabaseSaltBlock)saltBlock
-                                    keySpecBlock:(YapDatabaseKeySpecBlock)keySpecBlock;
+                                         options:(YapDatabaseOptions *)options
+                                 recordSaltBlock:(YapRecordDatabaseSaltBlock)recordSaltBlock;
 
 // This method can be used to derive a SQLCipher "key spec" from a
 // database password and salt.  Key spec derivation is somewhat costly.
 // The key spec is needed every time the database file is opened
-// (including every time YapDatabse makes a new database connection),
+// (including every time YapDatabase makes a new database connection),
 // So it benefits performance to pass a pre-derived key spec to
 // YapDatabase.
-+ (nullable NSData *)databaseKeySpecForPassword:(NSData *)passwordData saltData:(NSData *)saltData;
++ (nullable NSData *)deriveDatabaseKeySpecForPassword:(NSData *)passwordData saltData:(NSData *)saltData;
 
 #pragma mark - Utils
 

YapDatabase/Utilities/YapDatabaseCryptoUtils.m

@@ -140,7 +140,7 @@ NSError *YDBErrorWithDescription(NSString *description)
 
     if (![[NSFileManager defaultManager] fileExistsAtPath:databaseFilePath]) {
         YDBLogVerbose(@"%@ database file not found.", self.logTag);
-        return nil;
+        return NO;
     }
 
     NSData *headerData = [self readFirstNBytesOfDatabaseFile:databaseFilePath byteCount:kSqliteHeaderLength];
@@ -160,28 +160,30 @@ NSError *YDBErrorWithDescription(NSString *description)
 
 + (nullable NSError *)convertDatabaseIfNecessary:(NSString *)databaseFilePath
                                 databasePassword:(NSData *)databasePassword
-                                       saltBlock:(YapDatabaseSaltBlock)saltBlock
-                                    keySpecBlock:(YapDatabaseKeySpecBlock)keySpecBlock
+                                         options:(YapDatabaseOptions *)options
+                                 recordSaltBlock:(YapRecordDatabaseSaltBlock)recordSaltBlock
 {
     if (![self doesDatabaseNeedToBeConverted:databaseFilePath]) {
+        YDBLogInfo(@"%@ convertDatabaseIfNecessary: database does not need to be converted.", self.logTag);
         return nil;
     }
 
     return [self convertDatabase:databaseFilePath
                 databasePassword:databasePassword
-                       saltBlock:saltBlock
-                    keySpecBlock:keySpecBlock];
+                         options:options
+                 recordSaltBlock:recordSaltBlock];
 }
 
 + (nullable NSError *)convertDatabase:(NSString *)databaseFilePath
                      databasePassword:(NSData *)databasePassword
-                            saltBlock:(YapDatabaseSaltBlock)saltBlock
-                         keySpecBlock:(YapDatabaseKeySpecBlock)keySpecBlock
+                              options:(YapDatabaseOptions *)options
+                      recordSaltBlock:(YapRecordDatabaseSaltBlock)recordSaltBlock
 {
     YapAssert(databaseFilePath.length > 0);
     YapAssert(databasePassword.length > 0);
-    YapAssert(saltBlock);
-    YapAssert(keySpecBlock);
+    YapAssert(recordSaltBlock);
+
+    YDBLogInfo(@"%@ convertDatabase.", self.logTag);
 
     NSData *saltData;
     {
@@ -194,23 +196,15 @@ NSError *YDBErrorWithDescription(NSString *description)
         // Make sure we successfully persist the salt (persumably in the keychain) before
         // proceeding with the database conversion or we could leave the app in an
         // unrecoverable state.
-        saltBlock(saltData);
-    }
-
-    {
-        NSData *_Nullable keySpecData = [self databaseKeySpecForPassword:databasePassword saltData:saltData];
-        if (!keySpecData || keySpecData.length != kSQLCipherKeySpecLength) {
-            YDBLogError(@"Error deriving key spec");
-            return YDBErrorWithDescription(@"Invalid key spec");
+        YDBLogInfo(@"%@ convertDatabase: salt extracted.", self.logTag);
+        BOOL success = recordSaltBlock(saltData);
+        if (!success) {
+            YDBLogError(@"Failed to record salt, aborting conversion");
+            return YDBErrorWithDescription(@"Failed to record salt");
         }
-
-        YapAssert(keySpecData.length == kSQLCipherKeySpecLength);
-
-        // Make sure we successfully persist the key spec (persumably in the keychain) before
-        // proceeding with the database conversion or we could leave the app in an
-        // unrecoverable state.
-        keySpecBlock(keySpecData);
     }
+    
+    YDBLogInfo(@"%@ convertDatabase: key spec derived.", self.logTag);
 
     // -----------------------------------------------------------
     //
@@ -235,6 +229,8 @@ NSError *YDBErrorWithDescription(NSString *description)
             return YDBErrorWithDescription(@"Failed to open database");
         }
     }
+    
+    YDBLogInfo(@"%@ convertDatabase: database open.", self.logTag);
 
     // -----------------------------------------------------------
     //
@@ -248,6 +244,29 @@ NSError *YDBErrorWithDescription(NSString *description)
             return YDBErrorWithDescription(@"Failed to set SQLCipher key");
         }
     }
+    
+    YDBLogInfo(@"%@ convertDatabase: database keyed.", self.logTag);
+
+    if (options.legacyCipherCompatibilityVersion > 0) {
+        // `PRAGMA cipher_compatibility` is not available until SQLCipher 4.0.1 which corresponds
+        // to this version of sqlite.
+        // https://www.zetetic.net/blog/2018/12/18/sqlcipher-401-release/
+        const NSUInteger sqlCipherVersionSupportingCipherCompatibility = 3026000;
+        if (SQLITE_VERSION_NUMBER < sqlCipherVersionSupportingCipherCompatibility) {
+            YDBLogWarn(@"setting legacyCipherCompatibilityVersion on unsupported SQLCipher version has no effect.");
+        } else {
+            char *errorMsg;
+            NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA cipher_compatibility = %lu", (unsigned long)options.legacyCipherCompatibilityVersion];
+
+            if (sqlite3_exec(db, [pragmaCommand UTF8String], NULL, NULL, &errorMsg) != SQLITE_OK) {
+                NSString *errorString = [NSString stringWithFormat:@"failed to set database legacy cipher compatibility version: %s", errorMsg];
+                YDBLogError(@"%@", errorString);
+                YDBErrorWithDescription(errorString);
+            }
+
+            YDBLogVerbose(@"set database legacy cipher compatibility version: %lu", (unsigned long)options.legacyCipherCompatibilityVersion);
+        }
+    }
 
     // -----------------------------------------------------------
     //
@@ -302,6 +321,8 @@ NSError *YDBErrorWithDescription(NSString *description)
         // END DB setup copied from YapDatabase
         // BEGIN SQLCipher migration
     }
+    
+    YDBLogInfo(@"%@ convertDatabase: database configured.", self.logTag);
 
 #ifdef DEBUG
     // We can obtain the database salt in two ways: by reading the first 16 bytes of the encrypted
@@ -313,6 +334,8 @@ NSError *YDBErrorWithDescription(NSString *description)
 
         YapAssert([[self hexadecimalStringForData:saltData] isEqualToString:saltString]);
     }
+    
+    YDBLogInfo(@"%@ convertDatabase: salt confirmed.", self.logTag);
 #endif
 
     // -----------------------------------------------------------
@@ -327,6 +350,8 @@ NSError *YDBErrorWithDescription(NSString *description)
         if (error) {
             return error;
         }
+        
+        YDBLogInfo(@"%@ convertDatabase: encrypted header configured.", self.logTag);
 
         // Modify the first page, so that SQLCipher will overwrite, respecting our new cipher_plaintext_header_size
         NSString *tableName = [NSString stringWithFormat:@"signal-migration-%@", [NSUUID new].UUIDString];
@@ -340,6 +365,8 @@ NSError *YDBErrorWithDescription(NSString *description)
         if (error) {
             return error;
         }
+        
+        YDBLogInfo(@"%@ convertDatabase: database dirtied.", self.logTag);
 
         // Force a checkpoint so that the plaintext is written to the actual DB file, not just living in the WAL.
         int log, ckpt;
@@ -348,8 +375,12 @@ NSError *YDBErrorWithDescription(NSString *description)
             YDBLogError(@"%@ Error forcing checkpoint. status: %d, log: %d, ckpt: %d, error: %s", self.logTag, status, log, ckpt, sqlite3_errmsg(db));
             return YDBErrorWithDescription(@"Error forcing checkpoint.");
         }
+        
+        YDBLogInfo(@"%@ convertDatabase: checkpoint completed.", self.logTag);
 
         sqlite3_close(db);
+        
+        YDBLogInfo(@"%@ convertDatabase: database closed.", self.logTag);
     }
 
     return nil;
@@ -408,9 +439,13 @@ NSError *YDBErrorWithDescription(NSString *description)
     YapAssert(passwordData.length > 0);
     YapAssert(saltData.length == kSQLCipherSaltLength);
 
-    unsigned char *derivedKeyBytes = malloc((size_t)kSQLCipherDerivedKeyLength);
-    YapAssert(derivedKeyBytes);
-    // See: PBKDF2_ITER.
+    NSMutableData *_Nullable derivedKeyData = [NSMutableData dataWithLength:kSQLCipherDerivedKeyLength];
+    if (!derivedKeyData) {
+        YapFail(@"failed to allocate derivedKeyData");
+        return nil;
+    }
+
+    // See: PBKDF2_ITER from SQLCipher.
     const unsigned int workfactor = 64000;
 
     int result = CCKeyDerivationPBKDF(kCCPBKDF2,
@@ -420,23 +455,18 @@ NSError *YDBErrorWithDescription(NSString *description)
         (size_t)saltData.length,
         kCCPRFHmacAlgSHA1,
         workfactor,
-        derivedKeyBytes,
-        kSQLCipherDerivedKeyLength);
+        derivedKeyData.mutableBytes,
+        (size_t)derivedKeyData.length);
+
     if (result != kCCSuccess) {
         YDBLogError(@"Error deriving key: %d", result);
         return nil;
     }
 
-    NSData *_Nullable derivedKeyData = [NSData dataWithBytes:derivedKeyBytes length:kSQLCipherDerivedKeyLength];
-    if (!derivedKeyData || derivedKeyData.length != kSQLCipherDerivedKeyLength) {
-        YDBLogError(@"Invalid derived key: %d", result);
-        return nil;
-    }
-
-    return derivedKeyData;
+    return [derivedKeyData copy];
 }
 
-+ (nullable NSData *)databaseKeySpecForPassword:(NSData *)passwordData saltData:(NSData *)saltData
++ (nullable NSData *)deriveDatabaseKeySpecForPassword:(NSData *)passwordData saltData:(NSData *)saltData
 {
     YapAssert(passwordData.length > 0);
     YapAssert(saltData.length == kSQLCipherSaltLength);
@@ -452,7 +482,7 @@ NSError *YDBErrorWithDescription(NSString *description)
 
     YapAssert(keySpecData.length == kSQLCipherKeySpecLength);
 
-    return keySpecData;
+    return [keySpecData copy];
 }
 
 #pragma mark - Utils

YapDatabase/YapDatabase.m

@@ -7,6 +7,7 @@
 #import "YapDatabaseConnectionState.h"
 #import "YapDatabaseLogging.h"
 #import "YapDatabaseString.h"
+#import "YapDatabaseCryptoUtils.h"
 
 #import "sqlite3.h"
 
@@ -830,18 +831,49 @@ static int connectionBusyHandler(void *ptr, int count) {
 **/
 - (BOOL)configureEncryptionForDatabase:(sqlite3 *)sqlite
 {
+    if (options.cipherUnencryptedHeaderLength > 0) {
+        if (options.cipherKeySpecBlock)
+        {
+            // Do nothing.
+        } else if (!(options.cipherKeyBlock && options.cipherSaltBlock)) {
+            NSAssert(NO, @"If you're using YapDatabaseOptions.cipherUnencryptedHeaderLength, you need to set either cipherKeySpecBlock or both cipherKeyBlock and cipherSaltBlock.");
+            return NO;
+        }
+    }        
+
     if (options.cipherKeyBlock ||
         options.cipherKeySpecBlock)
 	{
         NSData *_Nullable keyData = nil;
         if (options.cipherKeySpecBlock)
         {
-            keyData = options.cipherKeySpecBlock();
-            if (!keyData)
+            if (options.cipherKeyBlock) {
+                NSAssert(NO, @"If you're using YapDatabaseOptions.cipherKeySpecBlock, you don't need to set a cipherKeySpecBlock.");
+                return NO;
+            }
+            if (options.cipherSaltBlock) {
+                NSAssert(NO, @"If you're using YapDatabaseOptions.cipherKeySpecBlock, you don't need to set a cipherSaltBlock.");
+                return NO;
+            }
+
+            NSData *_Nullable keySpecData = options.cipherKeySpecBlock();
+            if (!keySpecData)
             {
                 NSAssert(NO, @"YapDatabaseOptions.cipherKeySpecBlock cannot return nil!");
                 return NO;
             }
+            if (keySpecData.length != kSQLCipherKeySpecLength) {
+                NSAssert(NO, @"YapDatabaseOptions.cipherKeySpecBlock returned a key spec of unexpected length: %zd.", keySpecData.length);
+                return NO;
+            }
+
+            // Use a raw key spec, where the 96 hexadecimal digits are provided
+            // (i.e. 64 hex for the 256 bit key, followed by 32 hex for the 128 bit salt)
+            // using explicit BLOB syntax, e.g.:
+            //
+            // x'98483C6EB40B6C31A448C22A66DED3B5E5E8D5119CAC8327B655C8B5C483648101010101010101010101010101010101'
+            NSString *keySpecString = [NSString stringWithFormat:@"x'%@'", [self hexadecimalStringForData:keySpecData]];
+            keyData = [keySpecString dataUsingEncoding:NSUTF8StringEncoding];
         } else {
             keyData = options.cipherKeyBlock();
             if (!keyData)
@@ -850,63 +882,52 @@ static int connectionBusyHandler(void *ptr, int count) {
                 return NO;
             }
         }
-        
-        //Setting the PBKDF2 default iteration number (this will have effect next time database is opened)
-        if (options.cipherDefaultkdfIterNumber > 0) {
-            char *errorMsg;
-            NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA cipher_default_kdf_iter = %lu", (unsigned long)options.cipherDefaultkdfIterNumber];
-            if (sqlite3_exec(sqlite, [pragmaCommand UTF8String], NULL, NULL, &errorMsg) != SQLITE_OK)
-            {
-                YDBLogError(@"failed to set database cipher_default_kdf_iter: %s", errorMsg);
-                return NO;
+
+        BOOL hasCustomCipherConfiguration = NO;
+        if (options.cipherDefaultkdfIterNumber > 0 || options.kdfIterNumber > 0 || options.cipherPageSize) {
+            hasCustomCipherConfiguration = YES;
+
+            //Setting the PBKDF2 default iteration number (this will have effect next time database is opened)
+            if (options.cipherDefaultkdfIterNumber > 0) {
+                char *errorMsg;
+                NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA cipher_default_kdf_iter = %lu", (unsigned long)options.cipherDefaultkdfIterNumber];
+                if (sqlite3_exec(sqlite, [pragmaCommand UTF8String], NULL, NULL, &errorMsg) != SQLITE_OK)
+                {
+                    YDBLogError(@"failed to set database cipher_default_kdf_iter: %s", errorMsg);
+                    return NO;
+                }
+            }
+
+            //Setting the PBKDF2 iteration number
+            if (options.kdfIterNumber > 0) {
+                char *errorMsg;
+                NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA kdf_iter = %lu", (unsigned long)options.kdfIterNumber];
+                if (sqlite3_exec(sqlite, [pragmaCommand UTF8String], NULL, NULL, &errorMsg) != SQLITE_OK)
+                {
+                    YDBLogError(@"failed to set database kdf_iter: %s", errorMsg);
+                    return NO;
+                }
+            }
+
+            //Setting the encrypted database page size
+            if (options.cipherPageSize > 0) {
+                char *errorMsg;
+                NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA cipher_page_size = %lu", (unsigned long)options.cipherPageSize];
+                if (sqlite3_exec(sqlite, [pragmaCommand UTF8String], NULL, NULL, &errorMsg) != SQLITE_OK)
+                {
+                    YDBLogError(@"failed to set database cipher_page_size: %s", errorMsg);
+                    return NO;
+                }
             }
         }
         
-        //Setting the PBKDF2 iteration number
-        if (options.kdfIterNumber > 0) {
-            char *errorMsg;
-            NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA kdf_iter = %lu", (unsigned long)options.kdfIterNumber];
-            if (sqlite3_exec(sqlite, [pragmaCommand UTF8String], NULL, NULL, &errorMsg) != SQLITE_OK)
-            {
-                YDBLogError(@"failed to set database kdf_iter: %s", errorMsg);
-                return NO;
-            }
+        int status = sqlite3_key(sqlite, [keyData bytes], (int)[keyData length]);
+        if (status != SQLITE_OK)
+        {
+            YDBLogError(@"Error setting SQLCipher key: %d %s", status, sqlite3_errmsg(sqlite));
+            return NO;
         }
-        
-        //Setting the encrypted database page size
-        if (options.cipherPageSize > 0) {
-            char *errorMsg;
-            NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA cipher_page_size = %lu", (unsigned long)options.cipherPageSize];
-            if (sqlite3_exec(sqlite, [pragmaCommand UTF8String], NULL, NULL, &errorMsg) != SQLITE_OK)
-            {
-                YDBLogError(@"failed to set database cipher_page_size: %s", errorMsg);
-                return NO;
-            }
-        }
-        
-        if (options.cipherKeySpecBlock) {
-            // Use a raw key spec, where the 96 hexadecimal digits are provided
-            // (i.e. 64 hex for the 256 bit key, followed by 32 hex for the 128 bit salt)
-            // using explicit BLOB syntax, e.g.:
-            //
-            // x'98483C6EB40B6C31A448C22A66DED3B5E5E8D5119CAC8327B655C8B5C483648101010101010101010101010101010101'
-            NSString *keySpecString = [NSString stringWithFormat:@"x'%@'", [self hexadecimalStringForData:keyData]];
-            NSData *keySpecStringData = [keySpecString dataUsingEncoding:NSUTF8StringEncoding];
-            int status = sqlite3_key(sqlite, [keySpecStringData bytes], (int)[keySpecStringData length]);
-            if (status != SQLITE_OK)
-            {
-                YDBLogError(@"Error setting SQLCipher key: %d %s", status, sqlite3_errmsg(sqlite));
-                return NO;
-            }
-        } else {
-            int status = sqlite3_key(sqlite, [keyData bytes], (int)[keyData length]);
-            if (status != SQLITE_OK)
-            {
-                YDBLogError(@"Error setting SQLCipher key: %d %s", status, sqlite3_errmsg(sqlite));
-                return NO;
-            }
-        }
-        
+
         if (options.cipherUnencryptedHeaderLength > 0 &&
             (options.cipherKeySpecBlock ||
              options.cipherSaltBlock)) {
@@ -923,6 +944,10 @@ static int connectionBusyHandler(void *ptr, int count) {
                     NSAssert(NO, @"YapDatabaseOptions.cipherSaltBlock cannot return nil!");
                     return NO;
                 }
+                if (saltData.length != kSQLCipherSaltLength) {
+                    NSAssert(NO, @"YapDatabaseOptions.cipherSaltBlock returned a salt of unexpected length: %zd.", saltData.length);
+                    return NO;
+                }
 
                 {
                     char *errorMsg;
@@ -964,7 +989,30 @@ static int connectionBusyHandler(void *ptr, int count) {
                 return NO;
             }
         }
-	}
+
+
+        if (options.legacyCipherCompatibilityVersion > 0) {
+            // `PRAGMA cipher_compatibility` is not available until SQLCipher 4.0.1 which corresponds
+            // to this version of sqlite.
+            // https://www.zetetic.net/blog/2018/12/18/sqlcipher-401-release/
+            const NSUInteger sqlCipherVersionSupportingCipherCompatibility = 3026000;
+            if (SQLITE_VERSION_NUMBER < sqlCipherVersionSupportingCipherCompatibility) {
+                YDBLogWarn(@"setting legacyCipherCompatibilityVersion on unsupported SQLCipher version has no effect.");
+            } else {
+                NSAssert(!hasCustomCipherConfiguration, @"If you are specifying custom cipher parameters, you cannot use legacyCipherCompatibility. Instead you must specify *all* relevant cipher params. Details: https://discuss.zetetic.net/t/upgrading-to-sqlcipher-4/3283");
+                char *errorMsg;
+                NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA cipher_compatibility = %lu", (unsigned long)options.legacyCipherCompatibilityVersion];
+
+                if (sqlite3_exec(sqlite, [pragmaCommand UTF8String], NULL, NULL, &errorMsg) != SQLITE_OK) {
+                    YDBLogError(@"failed to set database legacy cipher compatibility version: %s", errorMsg);
+                    return NO;
+                }
+
+                YDBLogVerbose(@"set database legacy cipher compatibility version: %lu", (unsigned long)options.legacyCipherCompatibilityVersion);
+            }
+        }
+    }
+
 	
 	return YES;
 }
@@ -985,7 +1033,7 @@ static int connectionBusyHandler(void *ptr, int count) {
     return [hexString copy];
 }
 
-#endif
+#endif // SQLITE_HAS_CODEC
 
 /**
  * Creates the database tables we need:

YapDatabase/YapDatabaseConnection.h

@@ -294,6 +294,8 @@ typedef NS_OPTIONS(NSUInteger, YapDatabaseConnectionFlushMemoryFlags) {
 **/
 @property (atomic, assign, readonly) uint64_t pendingTransactionCount;
 
+@property (atomic) BOOL ignoreQueues;
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 #pragma mark Transactions
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

YapDatabase/YapDatabaseConnection.m

@@ -911,12 +911,16 @@ static int connectionBusyHandler(void *ptr, int count)
 		
 	#pragma clang diagnostic pop
 	};
-	
-	if (dispatch_get_specific(IsOnConnectionQueueKey))
-		block();
-	else
-		dispatch_sync(connectionQueue, block);
-	
+	 
+    if (self.ignoreQueues) {
+        block();
+    } else {
+        if (dispatch_get_specific(IsOnConnectionQueueKey))
+            block();
+        else
+            dispatch_sync(connectionQueue, block);
+    }
+    
 	return result;
 }
 
@@ -5136,11 +5140,15 @@ static int connectionBusyHandler(void *ptr, int count)
 	#pragma clang diagnostic pop
 	};
 	
-	if (dispatch_get_specific(IsOnConnectionQueueKey))
-		block();
-	else
-		dispatch_sync(connectionQueue, block);
-	
+    // NOTE: This should only be used during the migration, when we're already on the correct queue.
+    if (self.ignoreQueues) {
+        block();
+    } else {
+        if (dispatch_get_specific(IsOnConnectionQueueKey))
+            block();
+        else
+            dispatch_sync(connectionQueue, block);
+    }
 	return extConnection;
 }
 

YapDatabase/YapDatabaseOptions.h

@@ -244,7 +244,19 @@ typedef NSData *_Nonnull (^YapDatabaseCipherKeyBlock)(void);
 /**
  * Set a block here that returns the key spec (not the key) for the SQLCipher database.
  *
- * This key spec incorporates the "derived key" and the "salt".
+ * The key spec incorporates the "derived key" and the "salt".
+ *
+ * The key spec should be kSQLCipherKeySpecLength bytes in length.
+ *
+ * If you a key spec, you do NOT need to specify the salt (using cipherSaltBlock)
+ * and "key/password" (using cipherKeyBlock).
+ *
+ * For new databases, the key spec can be any N bytes where N is kSQLCipherKeySpecLength.
+ * You should consider generating them with SecRandomCopyBytes().
+ *
+ * For existing databases that were created using a "key/password" (i.e. cipherKeyBlock),
+ * you can derive a key spec using that key/password and the database's salt.  See
+ * comments in YapDatabaseCryptoUtils.h.
  *
  * This block allows you to fetch the key spec from the keychain (or elsewhere)
  * only when you need it, instead of persisting it in memory.
@@ -274,7 +286,26 @@ typedef NSData *_Nonnull (^YapDatabaseCipherKeyBlock)(void);
  **/
 @property (nonatomic, assign, readwrite) NSUInteger cipherUnencryptedHeaderLength;
 
-#endif
+/**
+ * If greater than zero, pegs the SQLCipher cipher parameters to the corresponding legacy version
+ * via `PRAGMA cipher_compatibility = 3;`
+ * https://www.zetetic.net/sqlcipher/sqlcipher-api/#cipher_compatibility
+ *
+ * Probably you should leave this as `0`, in which case, YapDB will automatically use
+ * `PRAGMA cipher_migrate` to migrate to the new, more secure, cipher configuration.
+ *
+ * https://www.zetetic.net/sqlcipher/sqlcipher-api/#cipher_migrate
+ *
+ * After migrating, you won't be able to read the database from an older major version of SQLCipher.
+ *
+ * If for some reason you do not want to migrate to the new more secure defaults, setting this to
+ * (e.g.) `3` would allow you to upgrade to SQLCipher v4.0.1+, while maintaining the legacy cipher
+ * configuration. This would allow you to continue to access the database from a legacy SQLite
+ * client.
+ */
+@property (nonatomic, assign, readwrite) NSUInteger legacyCipherCompatibilityVersion;
+
+#endif // SQLITE_HAS_CODEC
 
 /**
  * There are a few edge-case scenarios where the sqlite WAL (write-ahead log) file

YapDatabase/YapDatabaseOptions.m

@@ -27,8 +27,8 @@
 @synthesize cipherSaltBlock = cipherSaltBlock;
 @synthesize cipherKeySpecBlock = cipherKeySpecBlock;
 @synthesize cipherUnencryptedHeaderLength = cipherUnencryptedHeaderLength;
-
-#endif
+@synthesize legacyCipherCompatibilityVersion = legacyCipherCompatibilityVersion;
+#endif // SQLITE_HAS_CODEC
 @synthesize aggressiveWALTruncationSize = aggressiveWALTruncationSize;
 @synthesize enableMultiProcessSupport = enableMultiProcessSupport;
 
@@ -63,7 +63,8 @@
     copy->cipherSaltBlock = cipherSaltBlock;
     copy->cipherKeySpecBlock = cipherKeySpecBlock;
     copy->cipherUnencryptedHeaderLength = cipherUnencryptedHeaderLength;
-#endif
+    copy->legacyCipherCompatibilityVersion = legacyCipherCompatibilityVersion;
+#endif // SQLITE_HAS_CODEC
 	copy->aggressiveWALTruncationSize = aggressiveWALTruncationSize;
     copy->enableMultiProcessSupport = enableMultiProcessSupport;
 	

YapDatabase/YapDatabaseTransaction.m

@@ -25,6 +25,8 @@
 #endif
 #pragma unused(ydbLogLevel)
 
+typedef BOOL (^YapBoolBlock)(void);
+const NSUInteger kDefaultBatchSize = 1000;
 
 @implementation YapDatabaseReadTransaction
 
@@ -1569,24 +1571,30 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
+	__block BOOL stop = NO;
 	
 	// SELECT DISTINCT "collection" FROM "database2";
 	
 	int const column_idx_collection = SQLITE_COLUMN_START;
 	
-	int status;
-	while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-	{
+    YapDatabaseConnection *connection = self.connection;
+    __block int status;
+    [self whileLoopWithBatchSize:kDefaultBatchSize
+                       condition:^{
+                           if (stop || mutation.isMutated) {
+                               return NO;
+                           }
+                           
+                           status = sqlite3_step(statement);
+                           return (BOOL) (status == SQLITE_ROW);
+                       } loopBlock:^{
 		const unsigned char *text = sqlite3_column_text(statement, column_idx_collection);
 		int textSize = sqlite3_column_bytes(statement, column_idx_collection);
 		
 		NSString *collection = [[NSString alloc] initWithBytes:text length:textSize encoding:NSUTF8StringEncoding];
 		
 		block(collection, &stop);
-		
-		if (stop || mutation.isMutated) break;
-	}
+    }];
 	
 	if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 	{
@@ -2708,8 +2716,8 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
-	
+    __block BOOL stop = NO;
+
 	// SELECT "rowid", "key" FROM "database2" WHERE collection = ?;
 	
 	int const column_idx_rowid    = SQLITE_COLUMN_START + 0;
@@ -2719,9 +2727,17 @@
 	YapDatabaseString _collection; MakeYapDatabaseString(&_collection, collection);
 	sqlite3_bind_text(statement, bind_idx_collection, _collection.str, _collection.length, SQLITE_STATIC);
 	
-	int status;
-	while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-	{
+    YapDatabaseConnection *connection = self.connection;
+    __block int status;
+    [self whileLoopWithBatchSize:kDefaultBatchSize
+                       condition:^{
+                           if (stop || mutation.isMutated) {
+                               return NO;
+                           }
+                           
+                           status = sqlite3_step(statement);
+                           return (BOOL) (status == SQLITE_ROW);
+                       } loopBlock:^{
 		int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 		
 		const unsigned char *text = sqlite3_column_text(statement, column_idx_key);
@@ -2730,10 +2746,8 @@
 		NSString *key = [[NSString alloc] initWithBytes:text length:textSize encoding:NSUTF8StringEncoding];
 		
 		block(rowid, key, &stop);
-		
-		if (stop || mutation.isMutated) break;
-	}
-	
+                       }];
+
 	if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 	{
 		YDBLogError(@"%@ - sqlite_step error: %d %s", THIS_METHOD, status, sqlite3_errmsg(connection->db));
@@ -2765,8 +2779,8 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
-	
+    __block BOOL stop = NO;
+
 	// SELECT "rowid", "key" FROM "database2" WHERE collection = ?;
 	
 	int const column_idx_rowid    = SQLITE_COLUMN_START + 0;
@@ -2778,9 +2792,17 @@
 		YapDatabaseString _collection; MakeYapDatabaseString(&_collection, collection);
 		sqlite3_bind_text(statement, bind_idx_collection, _collection.str, _collection.length, SQLITE_STATIC);
 		
-		int status;
-		while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-		{
+        YapDatabaseConnection *connection = self.connection;
+        __block int status;
+        [self whileLoopWithBatchSize:kDefaultBatchSize
+                           condition:^{
+                               if (stop || mutation.isMutated) {
+                                   return NO;
+                               }
+                               
+                               status = sqlite3_step(statement);
+                               return (BOOL) (status == SQLITE_ROW);
+                           } loopBlock:^{
 			int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 			
 			const unsigned char *text = sqlite3_column_text(statement, column_idx_key);
@@ -2789,10 +2811,8 @@
 			NSString *key = [[NSString alloc] initWithBytes:text length:textSize encoding:NSUTF8StringEncoding];
 			
 			block(rowid, collection, key, &stop);
-			
-			if (stop || mutation.isMutated) break;
-		}
-		
+                           }];
+
 		if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 		{
 			YDBLogError(@"%@ - sqlite_step error: %d %s", THIS_METHOD, status, sqlite3_errmsg(connection->db));
@@ -2830,17 +2850,25 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
-	
+    __block BOOL stop = NO;
+
 	// SELECT "rowid", "collection", "key" FROM "database2";
 	
 	int const column_idx_rowid      = SQLITE_COLUMN_START + 0;
 	int const column_idx_collection = SQLITE_COLUMN_START + 1;
 	int const column_idx_key        = SQLITE_COLUMN_START + 2;
 	
-	int status;
-	while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-	{
+    YapDatabaseConnection *connection = self.connection;
+    __block int status;
+    [self whileLoopWithBatchSize:kDefaultBatchSize
+                       condition:^{
+                           if (stop || mutation.isMutated) {
+                               return NO;
+                           }
+                           
+                           status = sqlite3_step(statement);
+                           return (BOOL) (status == SQLITE_ROW);
+                       } loopBlock:^{
 		int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 		
 		const unsigned char *text1 = sqlite3_column_text(statement, column_idx_collection);
@@ -2855,10 +2883,8 @@
 		key        = [[NSString alloc] initWithBytes:text2 length:textSize2 encoding:NSUTF8StringEncoding];
 		
 		block(rowid, collection, key, &stop);
-		
-		if (stop || mutation.isMutated) break;
-	}
-	
+                       }];
+
 	if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 	{
 		YDBLogError(@"%@ - sqlite_step error: %d %s", THIS_METHOD, status, sqlite3_errmsg(connection->db));
@@ -2908,7 +2934,7 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
+	__block BOOL stop = NO;
 	
 	// SELECT "rowid", "key", "data", FROM "database2" WHERE "collection" = ?;
 	
@@ -2922,9 +2948,17 @@
 	
 	BOOL unlimitedObjectCacheLimit = (connection->objectCacheLimit == 0);
 	
-	int status;
-	while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-	{
+    YapDatabaseConnection *connection = self.connection;
+    __block int status;
+    [self whileLoopWithBatchSize:kDefaultBatchSize
+                       condition:^{
+                           if (stop || mutation.isMutated) {
+                               return NO;
+                           }
+                           
+                           status = sqlite3_step(statement);
+                           return (BOOL) (status == SQLITE_ROW);
+                       } loopBlock:^{
 		int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 		
 		const unsigned char *text = sqlite3_column_text(statement, column_idx_key);
@@ -2964,11 +2998,9 @@
 			}
 			
 			block(rowid, key, object, &stop);
-			
-			if (stop || mutation.isMutated) break;
 		}
-	}
-	
+                       }];
+
 	if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 	{
 		YDBLogError(@"%@ - sqlite_step error: %d %s", THIS_METHOD, status, sqlite3_errmsg(connection->db));
@@ -3019,7 +3051,7 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
+	__block BOOL stop = NO;
 	
 	BOOL unlimitedObjectCacheLimit = (connection->objectCacheLimit == 0);
 	
@@ -3035,9 +3067,18 @@
 		YapDatabaseString _collection; MakeYapDatabaseString(&_collection, collection);
 		sqlite3_bind_text(statement, bind_idx_collection, _collection.str, _collection.length, SQLITE_STATIC);
 		
-		int status;
-		while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-		{
+        YapDatabaseConnection *connection = self.connection;
+        __block int status;
+        [self whileLoopWithBatchSize:kDefaultBatchSize
+                           condition:^{
+                               if (stop || mutation.isMutated) {
+                                   return NO;
+                               }
+                               
+                               status = sqlite3_step(statement);
+                               return (BOOL) (status == SQLITE_ROW);
+                           } loopBlock:^{
+
 			int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 			
 			const unsigned char *text = sqlite3_column_text(statement, column_idx_key);
@@ -3078,11 +3119,9 @@
 				}
 				
 				block(rowid, collection, key, object, &stop);
-				
-				if (stop || mutation.isMutated) break;
 			}
-		}
-		
+        }];
+
 		if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 		{
 			YDBLogError(@"%@ - sqlite_step error: %d %s", THIS_METHOD, status, sqlite3_errmsg(connection->db));
@@ -3145,7 +3184,7 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
+	__block BOOL stop = NO;
 	
 	// SELECT "rowid", "collection", "key", "data" FROM "database2" ORDER BY \"collection\" ASC;";
 	
@@ -3156,9 +3195,18 @@
 	
 	BOOL unlimitedObjectCacheLimit = (connection->objectCacheLimit == 0);
 	
-	int status;
-	while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-	{
+    YapDatabaseConnection *connection = self.connection;
+    __block int status;
+    [self whileLoopWithBatchSize:kDefaultBatchSize
+                       condition:^{
+                           if (stop || mutation.isMutated) {
+                               return NO;
+                           }
+                           
+                           status = sqlite3_step(statement);
+                           return (BOOL) (status == SQLITE_ROW);
+                       } loopBlock:^{
+                           
 		int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 		
 		const unsigned char *text1 = sqlite3_column_text(statement, column_idx_collection);
@@ -3194,10 +3242,8 @@
 			}
 			
 			block(rowid, collection, key, object, &stop);
-			
-			if (stop || mutation.isMutated) break;
 		}
-	}
+    }];
 	
 	if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 	{
@@ -3212,6 +3258,30 @@
 	}
 }
 
+// Break loop cycles into batches, releasing stale objects
+// after each batch to avoid out of memory errors.
+- (void)whileLoopWithBatchSize:(NSUInteger)batchSize
+                     condition:(YapBoolBlock)conditionBlock
+                     loopBlock:(dispatch_block_t)loopBlock
+{
+    NSUInteger batchIndex = 0;
+    while (YES) {
+        if (batchIndex > 0) {
+            // This will log in DEBUG builds but not production.
+            YDBLogInfo(@"batchIndex: %lu (%lu)", (unsigned long) batchIndex, (unsigned long) (batchIndex * batchSize));
+        }
+        @autoreleasepool {
+            for (NSUInteger batchCounter = 0; batchCounter < batchSize; batchCounter++) {
+                if (!conditionBlock()) {
+                    return;
+                }
+                loopBlock();
+            }
+        }
+        batchIndex++;
+    }
+}
+
 /**
  * Fast enumeration over all keys and associated metadata in the given collection.
  * 
@@ -3250,8 +3320,8 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
-	
+    __block BOOL stop = NO;
+
 	// SELECT "rowid", "key", "metadata" FROM "database2" WHERE "collection" = ?;
 	
 	int const column_idx_rowid    = SQLITE_COLUMN_START + 0;
@@ -3264,9 +3334,17 @@
 	
 	BOOL unlimitedMetadataCacheLimit = (connection->metadataCacheLimit == 0);
 	
-	int status;
-	while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-	{
+    YapDatabaseConnection *connection = self.connection;
+    __block int status;
+    [self whileLoopWithBatchSize:kDefaultBatchSize
+                       condition:^{
+                           if (stop || mutation.isMutated) {
+                               return NO;
+                           }
+                           
+                           status = sqlite3_step(statement);
+                           return (BOOL) (status == SQLITE_ROW);
+                       } loopBlock:^{
 		int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 		
 		const unsigned char *text = sqlite3_column_text(statement, column_idx_key);
@@ -3317,10 +3395,8 @@
 			}
 			
 			block(rowid, key, metadata, &stop);
-			
-			if (stop || mutation.isMutated) break;
 		}
-	}
+                       }];
 	
 	if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 	{
@@ -3374,7 +3450,7 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
+	__block BOOL stop = NO;
 	
 	BOOL unlimitedMetadataCacheLimit = (connection->metadataCacheLimit == 0);
 	
@@ -3390,9 +3466,17 @@
 		YapDatabaseString _collection; MakeYapDatabaseString(&_collection, collection);
 		sqlite3_bind_text(statement, bind_idx_collection, _collection.str, _collection.length, SQLITE_STATIC);
 		
-		int status;
-		while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-		{
+        YapDatabaseConnection *connection = self.connection;
+        __block int status;
+        [self whileLoopWithBatchSize:kDefaultBatchSize
+                           condition:^{
+                               if (stop || mutation.isMutated) {
+                                   return NO;
+                               }
+                               
+                               status = sqlite3_step(statement);
+                               return (BOOL) (status == SQLITE_ROW);
+                           } loopBlock:^{
 			int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 			
 			const unsigned char *text = sqlite3_column_text(statement, column_idx_key);
@@ -3443,11 +3527,9 @@
 				}
 				
 				block(rowid, collection, key, metadata, &stop);
-				
-				if (stop || mutation.isMutated) break;
 			}
-		}
-		
+                           }];
+
 		if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 		{
 			YDBLogError(@"%@ - sqlite_step error: %d %s", THIS_METHOD, status, sqlite3_errmsg(connection->db));
@@ -3511,7 +3593,7 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
+	__block BOOL stop = NO;
 	
 	// SELECT "rowid", "collection", "key", "metadata" FROM "database2" ORDER BY "collection" ASC;
 	
@@ -3522,9 +3604,17 @@
 	
 	BOOL unlimitedMetadataCacheLimit = (connection->metadataCacheLimit == 0);
 	
-	int status;
-	while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-	{
+    YapDatabaseConnection *connection = self.connection;
+    __block int status;
+    [self whileLoopWithBatchSize:kDefaultBatchSize
+                       condition:^{
+                           if (stop || mutation.isMutated) {
+                               return NO;
+                           }
+                           
+                           status = sqlite3_step(statement);
+                           return (BOOL) (status == SQLITE_ROW);
+                       } loopBlock:^{
 		int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 		
 		const unsigned char *text1 = sqlite3_column_text(statement, column_idx_collection);
@@ -3581,11 +3671,9 @@
 			}
 			
 			block(rowid, collection, key, metadata, &stop);
-			
-			if (stop || mutation.isMutated) break;
 		}
-	}
-	
+                       }];
+
 	if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 	{
 		YDBLogError(@"%@ - sqlite_step error: %d %s", THIS_METHOD, status, sqlite3_errmsg(connection->db));
@@ -3635,7 +3723,7 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
+	__block BOOL stop = NO;
 	
 	// SELECT "rowid", "key", "data", "metadata" FROM "database2" WHERE "collection" = ?;
 	
@@ -3651,9 +3739,17 @@
 	BOOL unlimitedObjectCacheLimit = (connection->objectCacheLimit == 0);
 	BOOL unlimitedMetadataCacheLimit = (connection->metadataCacheLimit == 0);
 	
-	int status;
-	while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-	{
+    YapDatabaseConnection *connection = self.connection;
+    __block int status;
+    [self whileLoopWithBatchSize:kDefaultBatchSize
+                       condition:^{
+                           if (stop || mutation.isMutated) {
+                               return NO;
+                           }
+                           
+                           status = sqlite3_step(statement);
+                           return (BOOL) (status == SQLITE_ROW);
+                       } loopBlock:^{
 		int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 		
 		const unsigned char *text = sqlite3_column_text(statement, column_idx_key);
@@ -3730,11 +3826,9 @@
 			}
 			
 			block(rowid, key, object, metadata, &stop);
-			
-			if (stop || mutation.isMutated) break;
 		}
-	}
-	
+                       }];
+
 	if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 	{
 		YDBLogError(@"%@ - sqlite_step error: %d %s", THIS_METHOD, status, sqlite3_errmsg(connection->db));
@@ -3785,7 +3879,7 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
+	__block BOOL stop = NO;
 	
 	BOOL unlimitedObjectCacheLimit = (connection->objectCacheLimit == 0);
 	BOOL unlimitedMetadataCacheLimit = (connection->metadataCacheLimit == 0);
@@ -3803,9 +3897,17 @@
 		YapDatabaseString _collection; MakeYapDatabaseString(&_collection, collection);
 		sqlite3_bind_text(statement, bind_idx_collection, _collection.str, _collection.length, SQLITE_STATIC);
 		
-		int status;
-		while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-		{
+        YapDatabaseConnection *connection = self.connection;
+        __block int status;
+        [self whileLoopWithBatchSize:kDefaultBatchSize
+                           condition:^{
+                               if (stop || mutation.isMutated) {
+                                   return NO;
+                               }
+                               
+                               status = sqlite3_step(statement);
+                               return (BOOL) (status == SQLITE_ROW);
+                           } loopBlock:^{
 			int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 			
 			const unsigned char *text = sqlite3_column_text(statement, column_idx_key);
@@ -3883,11 +3985,9 @@
 				}
 				
 				block(rowid, collection, key, object, metadata, &stop);
-				
-				if (stop || mutation.isMutated) break;
 			}
-		}
-		
+                           }];
+
 		if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 		{
 			YDBLogError(@"%@ - sqlite_step error: %d %s", THIS_METHOD, status, sqlite3_errmsg(connection->db));
@@ -3950,7 +4050,7 @@
 	if (statement == NULL) return;
 	
 	YapMutationStackItem_Bool *mutation = [connection->mutationStack push]; // mutation during enumeration protection
-	BOOL stop = NO;
+	__block BOOL stop = NO;
 	
 	// SELECT "rowid", "collection", "key", "data", "metadata" FROM "database2" ORDER BY \"collection\" ASC;";
 	
@@ -3963,9 +4063,17 @@
 	BOOL unlimitedObjectCacheLimit = (connection->objectCacheLimit == 0);
 	BOOL unlimitedMetadataCacheLimit = (connection->metadataCacheLimit == 0);
 	
-	int status;
-	while ((status = sqlite3_step(statement)) == SQLITE_ROW)
-	{
+    YapDatabaseConnection *connection = self.connection;
+    __block int status;
+    [self whileLoopWithBatchSize:kDefaultBatchSize
+                       condition:^{
+                           if (stop || mutation.isMutated) {
+                               return NO;
+                           }
+                           
+                           status = sqlite3_step(statement);
+                           return (BOOL) (status == SQLITE_ROW);
+                       } loopBlock:^{
 		int64_t rowid = sqlite3_column_int64(statement, column_idx_rowid);
 		
 		const unsigned char *text1 = sqlite3_column_text(statement, column_idx_collection);
@@ -4028,11 +4136,9 @@
 			}
 			
 			block(rowid, collection, key, object, metadata, &stop);
-			
-			if (stop || mutation.isMutated) break;
 		}
-	}
-	
+                       }];
+
 	if ((status != SQLITE_DONE) && !stop && !mutation.isMutated)
 	{
 		YDBLogError(@"%@ - sqlite_step error: %d %s", THIS_METHOD, status, sqlite3_errmsg(connection->db));